[PROTO-1513] Setup e2e test stub with LocalStack, wire it up to CI, a…

…nd automate Mongo init (#7705)

.circleci/src/workflows/ddex-stage.yml

@@ -1,5 +1,13 @@
 when: << pipeline.parameters.run-ddex-stage-workflow >>
 jobs:
+  - test:
+      name: test-ddex-e2e
+      context: Vercel
+      service: ddex-e2e
+  - test:
+      name: test-ddex-unittests
+      context: Vercel
+      service: ddex-unittests
   - push-docker-image:
       name: push-ddex-web
       context: [Vercel, dockerhub]
@@ -24,6 +32,8 @@ jobs:
   - deploy-stage-nodes:
       name: deploy-stage-ddex
       requires:
+        - test-ddex-e2e
+        - test-ddex-unittests
         - push-ddex-web
         - push-ddex-ingester
         - push-ddex-publisher

.circleci/src/workflows/ddex-webapp.yml

@@ -1,3 +1,11 @@
 when: << pipeline.parameters.run-ddex-webapp-workflow >>
 jobs:
   - ddex-webapp-init
+  - test:
+      name: test-ddex-e2e
+      context: Vercel
+      service: ddex-e2e
+  - test:
+      name: test-ddex-unittests
+      context: Vercel
+      service: ddex-unittests

dev-tools/audius-compose

@@ -435,6 +435,7 @@ def load_env(protocol_dir, service, environment):
 @click.option("-e", "--elasticsearch-replicas", default=1, type=int)
 @click.option("-b", "--block-explorer", is_flag=True)
 @click.option("-o", "--anti-abuse-oracle", is_flag=True)
+@click.option("--dashboard", is_flag=True, help="Bring up the protocol dashboard container")
 @click.option(
     "-n",
     "--notifs",
@@ -448,7 +449,7 @@ def load_env(protocol_dir, service, environment):
     help="Bring up the pedalboard containers",
 )
 @click.option("--comms", is_flag=True, help="Bring up comms/dms containers")
-@click.option("--ddex", is_flag=True, help="Bring up ddex containers")
+@click.option("--ddex-only", is_flag=True, help="Bring up only ddex containers")
 @click.option("-a", "--args", type=str, multiple=True)
 @click.option(
     "--prod",
@@ -465,10 +466,11 @@ def up(
     elasticsearch_replicas,
     block_explorer,
     anti_abuse_oracle,
+    dashboard,
     notifs,
     pedalboard,
     comms,
-    ddex,
+    ddex_only,
     args,
     prod,
     services,
@@ -477,10 +479,28 @@ def up(
         protocol_dir, discovery_provider_replicas, elasticsearch_replicas, prod
     )
 
-    AAO_DIR = pathlib.Path(os.getenv("AAO_DIR", protocol_dir / "../anti-abuse-oracle"))
-
-    # Mediorum's Dockerfile.prod doesn't support multiple mediorum instances, so always use dev
-    os.environ['MEDIORUM_ENV_TYPE'] = 'dev'
+    if ddex_only:
+        generate_ddex_mongo_key(protocol_dir)
+        profiles = ["--profile=ddex"]
+    else:
+        AAO_DIR = pathlib.Path(os.getenv("AAO_DIR", protocol_dir / "../anti-abuse-oracle"))
+        os.environ['MEDIORUM_ENV_TYPE'] = 'dev'  # Mediorum's Dockerfile.prod doesn't support multiple mediorum instances, so always use dev
+        profiles = (
+            *(["--profile=block-explorer"] if block_explorer else []),
+            *(["--profile=elasticsearch"] if elasticsearch_replicas else []),
+            *(["--profile=notifications"] if notifs else []),
+            *(["--profile=pedalboard"] if pedalboard else []),
+            *(["--profile=comms"] if comms else []),
+            *(["--profile=dashboard"] if dashboard else []),
+            "--profile=storage",
+            "--profile=libs",
+            "--profile=solana",
+            "--profile=identity",
+            "--profile=discovery",
+            "--profile=chains",
+            "--profile=healthz",
+            *(["--file=" + str(AAO_DIR / 'audius-compose.yml')] if anti_abuse_oracle else []),
+        )
 
     ctx.invoke(pull)
     sys.exit(
@@ -491,20 +511,7 @@ def up(
                 f"--project-directory={protocol_dir / 'dev-tools/compose'}",
                 f"--project-name={protocol_dir.name}",
                 f"--file={protocol_dir / 'dev-tools/compose/docker-compose.yml'}",
-                *(
-                    [f"--file={AAO_DIR / 'audius-compose.yml'}"]
-                    if anti_abuse_oracle
-                    else []
-                ),
-                *(["--profile=block-explorer"] if block_explorer else []),
-                *(["--profile=elasticsearch"] if elasticsearch_replicas else []),
-                *(["--profile=notifications"] if notifs else []),
-                *(["--profile=pedalboard"] if pedalboard else []),
-                *(["--profile=comms"] if comms else []),
-                *(["--profile=ddex"] if ddex else []),
-                *(["--profile=storage"]),
-                *(["--profile=libs"]),
-                *(["--profile=solana"]),
+                *profiles,
                 "up",
                 "--build",
                 "--detach",
@@ -583,6 +590,23 @@ def test_build(protocol_dir):
     )
 
 
+def generate_ddex_mongo_key(protocol_dir):
+    keyfile_path = (protocol_dir / "packages/ddex/mongo-keyfile")
+    if not keyfile_path.exists():
+        result = subprocess.run(
+            ["openssl", "rand", "-base64", "756"],
+            capture_output=True,
+            text=True
+        )
+        key_content = result.stdout
+
+        with open(keyfile_path, "w") as keyfile:
+            keyfile.write(key_content)
+
+        subprocess.run(
+              ["chmod", "400", "packages/ddex/mongo-keyfile"],
+        )
+
 @test.command(name="run", default_command=True)
 @click.argument("service")
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
@@ -592,6 +616,11 @@ def test_run(protocol_dir, service, args):
     Runs tests for a given service.
     """
     generate_env(protocol_dir, 0, 0)
+    env = os.environ.copy()
+
+    if service.startswith("ddex"):
+        env["COMPOSE_PROFILES"] = "ddex"
+        generate_ddex_mongo_key(protocol_dir)
 
     sys.exit(
         subprocess.run(
@@ -607,7 +636,8 @@ def test_run(protocol_dir, service, args):
                 f"test-{service}",
                 "test",
                 *args,
-            ]
+            ],
+            env=env
         ).returncode
     )
 

dev-tools/compose/docker-compose.blockchain.yml

@@ -17,6 +17,8 @@ services:
       - '8545:8545'
     deploy:
       mode: global
+    profiles:
+      - chains
 
   poa-blockscout-db:
     image: postgres:13.6
@@ -71,6 +73,8 @@ services:
       - '8546:8545'
     deploy:
       mode: global
+    profiles:
+      - chains
 
   eth-blockscout-db:
     image: postgres:13.6

dev-tools/compose/docker-compose.ddex.yml

@@ -14,6 +14,9 @@ services:
       - DDEX_PORT=9000
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     env_file: .env
+    depends_on:
+      ddex-mongo-init:
+        condition: service_completed_successfully
     ports:
       - "9000:9000"
     networks:
@@ -30,9 +33,14 @@ services:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     env_file: .env
     depends_on:
-      ddex-mongo:
-        condition: service_healthy
+      ddex-mongo-init:
+        condition: service_completed_successfully
     entrypoint: ./ingester --service crawler
+    healthcheck:
+      test: ["CMD-SHELL", "pgrep ./ingester || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
     networks:
       - ddex-network
     profiles:
@@ -47,9 +55,14 @@ services:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     env_file: .env
     depends_on:
-      ddex-mongo:
-        condition: service_healthy
+      ddex-mongo-init:
+        condition: service_completed_successfully
     entrypoint: ./ingester --service indexer
+    healthcheck:
+      test: ["CMD-SHELL", "pgrep ./ingester || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
     networks:
       - ddex-network
     profiles:
@@ -64,9 +77,14 @@ services:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     env_file: .env
     depends_on:
-      ddex-mongo:
-        condition: service_healthy
+      ddex-mongo-init:
+        condition: service_completed_successfully
     entrypoint: ./ingester --service parser
+    healthcheck:
+      test: ["CMD-SHELL", "pgrep ./ingester || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
     networks:
       - ddex-network
     profiles:
@@ -85,10 +103,12 @@ services:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     env_file: .env
     depends_on:
-      ddex-mongo:
-        condition: service_healthy
+      ddex-mongo-init:
+        condition: service_completed_successfully
     ports:
       - "9001:9001"
+    healthcheck:
+      test: ["CMD-SHELL", "pgrep node || exit 1"]
     networks:
       - ddex-network
     profiles:
@@ -104,6 +124,12 @@ services:
     environment:
       - MONGO_INITDB_ROOT_USERNAME=mongo
       - MONGO_INITDB_ROOT_PASSWORD=mongo
+    entrypoint: >
+      bash -c '
+        chmod 400 /etc/mongo-keyfile
+        chown 999:999 /etc/mongo-keyfile
+        exec docker-entrypoint.sh "$$@"
+      '
     command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile
     ports:
       - "27017:27017"
@@ -114,14 +140,9 @@ services:
           if (!ping.ok) {
             exit(1);  // Unhealthy: MongoDB server not responding
           }
-          var result = rs.status();
-          if (result.ok && result.set) {
-            exit(0);  // Healthy: MongoDB server is up and replica set is initiated
-          } else {
-            exit(1);  // Unhealthy: Replica set not initiated or not OK
-          }
+          exit(0);  // Healthy: MongoDB server is up (replica set status will be checked by ddex-mongo-init)
         "' localhost:27017/ddex
-      interval: 30s
+      interval: 10s
       timeout: 10s
       retries: 5
       start_period: 40s
@@ -130,6 +151,45 @@ services:
     profiles:
       - ddex
 
+  ddex-mongo-init:
+    container_name: ddex-mongo-init
+    image: mongo:latest
+    restart: "no"
+    command: >
+      bash -c "
+        until mongosh --host ddex-mongo:27017 --authenticationDatabase admin -u mongo -p mongo --eval 'db.runCommand({ping: 1})'; do
+          echo 'Waiting for ddex-mongo...'
+          sleep 2
+        done
+        mongosh --host ddex-mongo:27017 --authenticationDatabase admin -u mongo -p mongo --eval '
+          var rsConfig = {
+            _id: \"rs0\",
+            members: [{ _id: 0, host: \"ddex-mongo:27017\" }]
+          };
+          if (db.isMaster().ismaster) {
+            print(\"Replica set already initiated.\");
+            quit(0);
+          }
+          else {
+            var result = rs.initiate(rsConfig);
+            if (result.ok) {
+              print(\"Replica set initiated.\");
+              quit(0);
+            }
+            else {
+              print(\"Failed to initiate replica set.\");
+              quit(1);
+            }
+          }
+        '
+      "
+    networks:
+      - ddex-network
+    depends_on:
+      - ddex-mongo
+    profiles:
+      - ddex
+
   ddex-ingester:
     container_name: ddex-ingester
     build:

dev-tools/compose/docker-compose.discovery.prod.yml

@@ -12,6 +12,8 @@ services:
     deploy:
       mode: replicated
       replicas: '${DISCOVERY_PROVIDER_REPLICAS}'
+    profiles:
+      - discovery
 
   discovery-provider-notifications:
     build:
@@ -44,6 +46,8 @@ services:
       context: ${PROJECT_ROOT}/packages/discovery-provider
       dockerfile: Dockerfile.nginx
     restart: unless-stopped
+    profiles:
+      - discovery
 
   comms:
     build:
@@ -62,6 +66,8 @@ services:
       replicas: '${DISCOVERY_PROVIDER_REPLICAS}'
     volumes:
       - ${PROJECT_ROOT}/dev-tools:/tmp/dev-tools
+    profiles:
+      - comms
 
   trpc:
     build:
@@ -71,6 +77,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+    profiles:
+      - discovery
 
   es-indexer:
     build:
@@ -83,6 +91,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+    profiles:
+      - discovery
 
   discovery-provider-elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0
@@ -171,3 +181,5 @@ services:
     deploy:
       mode: replicated
       replicas: '${DISCOVERY_PROVIDER_REPLICAS}'
+    profiles:
+      - discovery

dev-tools/compose/docker-compose.healthz.yml

@@ -2,7 +2,10 @@ version: "3.9"
 
 services:
   healthz:
+    container_name: healthz
     build:
       context: ${PROJECT_ROOT}/monitoring/healthz
     deploy:
       mode: global
+    profiles:
+      - healthz