Merge branch 'main' into dj-primary-stems-buttons

AudiusProject · Mar 12, 2024 · d81d101 · d81d101
2 parents 01b84ba + 5d8fd5b
commit d81d101
Show file tree

Hide file tree

Showing 148 changed files with 2,520 additions and 1,997 deletions.
diff --git a/.circleci/src/jobs/@web-jobs.yml b/.circleci/src/jobs/@web-jobs.yml
@@ -114,6 +114,22 @@ web-test-staging:
     - store_artifacts:
         path: /home/circleci/audius-protocol/packages/probers/cypress/screenshots
 
+web-test:
+  working_directory: ~/audius-protocol
+  docker:
+    - image: cimg/node:18.17
+  resource_class: xlarge
+  steps:
+    - checkout
+    - attach_workspace:
+        at: ./
+    - run:
+        name: test
+        no_output_timeout: 30m
+        command: |
+          cd packages/web
+          npm run test
+
 web-build-production:
   working_directory: ~/audius-protocol
   docker:

diff --git a/.circleci/src/workflows/web.yml b/.circleci/src/workflows/web.yml
@@ -57,6 +57,11 @@ jobs:
       requires:
         - web-build-staging
 
+  - web-test:
+      context: Audius Client
+      requires:
+        - web-build-staging
+
   - web-deploy-staging-s3:
       context: Audius Client
       requires:

diff --git a/dev-tools/audius-compose b/dev-tools/audius-compose
@@ -620,25 +620,36 @@ def test_run(protocol_dir, service, args):
         env["COMPOSE_PROFILES"] = "ddex"
         generate_ddex_mongo_key(protocol_dir)
 
-    sys.exit(
-        subprocess.run(
-            [
-                "docker",
-                "compose",
-                f"--file={protocol_dir / 'dev-tools/compose/docker-compose.test.yml'}",
-                f"--project-name={protocol_dir.name}-test",
-                f"--project-directory={protocol_dir / 'dev-tools/compose'}",
-                "run",
-                "--rm",
-                "--build",
-                f"test-{service}",
-                "test",
-                *args,
-            ],
-            env=env
-        ).returncode
+    result = subprocess.run(
+        [
+            "docker",
+            "compose",
+            f"--file={protocol_dir / 'dev-tools/compose/docker-compose.test.yml'}",
+            f"--project-name={protocol_dir.name}-test",
+            f"--project-directory={protocol_dir / 'dev-tools/compose'}",
+            "run",
+            "--rm",
+            "--build",
+            f"test-{service}",
+            "test",
+            *args,
+        ],
+        env=env
     )
 
+    if result.returncode != 0 and service.startswith("ddex"):
+        containers = ["ddex-crawler-release-by-release", "ddex-parser-release-by-release"] if service.endswith("release") else ["ddex-crawler-batched", "ddex-parser-batched"]
+        for container in containers:
+            logs_result = subprocess.run(f"docker logs {container}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if logs_result.returncode == 0:
+                print(f"\nLogs for {container}:\n")
+                print(logs_result.stdout.decode("utf-8"))
+            else:
+                print(f"\nFailed to retrieve logs for {container}\n")
+                print(logs_result.stderr.decode("utf-8"))
+
+    sys.exit(result.returncode)
+
 
 @test.command(name="up")
 @click.argument("service")

diff --git a/dev-tools/compose/docker-compose.ddex.yml b/dev-tools/compose/docker-compose.ddex.yml
@@ -46,28 +46,6 @@ services:
     profiles:
       - ddex
 
-  ddex-indexer:
-    container_name: ddex-indexer
-    build:
-      context: ${PROJECT_ROOT}/packages/ddex/ingester
-      dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile
-    environment:
-      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
-    env_file: .env
-    depends_on:
-      ddex-mongo-init:
-        condition: service_completed_successfully
-    entrypoint: ./ingester --service indexer
-    healthcheck:
-      test: ["CMD-SHELL", "pgrep ./ingester || exit 1"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-    networks:
-      - ddex-network
-    profiles:
-      - ddex
-
   ddex-parser:
     container_name: ddex-parser
     build:
@@ -190,6 +168,20 @@ services:
     profiles:
       - ddex
 
+  ddex-s3:
+    container_name: ddex-s3
+    env_file: .env
+    image: localstack/localstack:s3-latest
+    ports:
+      - "127.0.0.1:4566:4566"
+    networks:
+      - ddex-network
+    volumes:
+      - "ddex-s3:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    profiles:
+      - ddex
+
   ddex-ingester:
     container_name: ddex-ingester
     env_file: .env
@@ -213,3 +205,4 @@ networks:
 
 volumes:
   ddex-mongo-db:
+  ddex-s3:
diff --git a/dev-tools/compose/docker-compose.test.yml b/dev-tools/compose/docker-compose.test.yml
@@ -448,25 +448,10 @@ services:
       service: ddex-crawler
     container_name: ddex-crawler-release-by-release
     logging: *default-logging
-    environment:
-      AWS_ENDPOINT: 'http://ddex-s3-release-by-release:4566'
-      TEST_MODE: 'true'
-      DDEX_CHOREOGRAPHY: 'ERNReleaseByRelease'
-    depends_on:
-      ddex-mongo-init:
-        condition: service_completed_successfully
-      ddex-s3-release-by-release:
-        condition: service_healthy
-
-  ddex-indexer-release-by-release:
-    extends:
-      file: docker-compose.yml
-      service: ddex-indexer
-    container_name: ddex-indexer-release-by-release
-    logging: *default-logging
     environment:
       AWS_ENDPOINT: 'http://ddex-s3-release-by-release:4566'
       DDEX_CHOREOGRAPHY: 'ERNReleaseByRelease'
+      TEST_MODE: 'true'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -530,40 +515,23 @@ services:
     depends_on:
       ddex-crawler-release-by-release:
         condition: service_healthy
-      ddex-indexer-release-by-release:
-        condition: service_healthy
       ddex-parser-release-by-release:
         condition: service_healthy
       # ddex-publisher:
       #   condition: service_healthy
       # Leaving out publisher for now because it takes a long time to build.
       # We don't actually upload anything to Audius in the e2e test, but having a "dry run" publisher mode could be useful
-  
+
   ddex-crawler-batched:
     extends:
       file: docker-compose.yml
       service: ddex-crawler
     container_name: ddex-crawler-batched
     logging: *default-logging
-    environment:
-      AWS_ENDPOINT: 'http://ddex-s3-batched:4566'
-      TEST_MODE: 'true'
-      DDEX_CHOREOGRAPHY: 'ERNBatched'
-    depends_on:
-      ddex-mongo-init:
-        condition: service_completed_successfully
-      ddex-s3-batched:
-        condition: service_healthy
-
-  ddex-indexer-batched:
-    extends:
-      file: docker-compose.yml
-      service: ddex-indexer
-    container_name: ddex-indexer-batched
-    logging: *default-logging
     environment:
       AWS_ENDPOINT: 'http://ddex-s3-batched:4566'
       DDEX_CHOREOGRAPHY: 'ERNBatched'
+      TEST_MODE: 'true'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -626,8 +594,6 @@ services:
     depends_on:
       ddex-crawler-batched:
         condition: service_healthy
-      ddex-indexer-batched:
-        condition: service_healthy
       ddex-parser-batched:
         condition: service_healthy
 

diff --git a/dev-tools/compose/docker-compose.yml b/dev-tools/compose/docker-compose.yml
@@ -198,14 +198,6 @@ services:
       DDEX_CHOREOGRAPHY: 'ERNReleaseByRelease'
     <<: *common
 
-  ddex-indexer:
-    extends:
-      file: docker-compose.ddex.yml
-      service: ddex-indexer
-    environment:
-      DDEX_CHOREOGRAPHY: 'ERNReleaseByRelease'
-    <<: *common
-
   ddex-parser:
     extends:
       file: docker-compose.ddex.yml
@@ -233,6 +225,12 @@ services:
       file: docker-compose.ddex.yml
       service: ddex-mongo-init
     <<: *common
+
+  ddex-s3:
+    extends:
+      file: docker-compose.ddex.yml
+      service: ddex-s3
+    <<: *common
 
   ddex-ingester:
     extends:
@@ -311,3 +309,4 @@ volumes:
   mediorum:
   legacy_creator_file_storage:
   ddex-mongo-db:
+  ddex-s3:
diff --git a/dev-tools/config.json b/dev-tools/config.json
@@ -129,13 +129,15 @@
     "ETH_TOKEN_ADDRESS": "0xdcB2fC9469808630DD0744b0adf97C0003fC29B2",
     "ETH_REGISTRY_ADDRESS": "0xABbfF712977dB51f9f212B85e8A4904c818C2b63",
     "ETH_OWNER_WALLET": "0x855FA758c77D68a04990E992aA4dcdeF899F654A",
+    "AWS_ENDPOINT": "http://ddex-s3:4566",
     "AWS_ACCESS_KEY_ID": "test",
     "AWS_SECRET_ACCESS_KEY": "test",
     "AWS_REGION": "us-west-2",
     "AWS_BUCKET_RAW": "audius-test-raw",
-    "AWS_BUCKET_INDEXED": "audius-test-indexed",
+    "AWS_BUCKET_CRAWLED": "audius-test-crawled",
     "DDEX_KEY": "49d5e13d355709b615b7cce7369174fb240b6b39",
     "DDEX_SECRET": "2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152",
-    "SESSION_SECRET": "something random"
+    "SESSION_SECRET": "something random",
+    "NODE_ENV": "stage"
   }
 }
diff --git a/mediorum/.version.json b/mediorum/.version.json
@@ -1,4 +1,4 @@
 {
-  "version": "0.6.50",
+  "version": "0.6.57",
   "service": "content-node"
 }
diff --git a/mediorum/server/serve_blob.go b/mediorum/server/serve_blob.go
@@ -332,7 +332,7 @@ func (ss *MediorumServer) logTrackListen(c echo.Context) {
 	}
 
 	httpClient := http.Client{
-		Timeout: 5 * time.Second, // identity svc slow
+		Timeout: 1 * time.Minute, // identity svc slow
 	}
 
 	sig, err := signature.ParseFromQueryString(c.QueryParam("signature"))