[PROTO-1513] Add e2e ddex test

dev-tools/compose/docker-compose.ddex.yml

@@ -192,9 +192,21 @@ services:
 
   ddex-ingester:
     container_name: ddex-ingester
+    env_file: .env
+    environment:
+      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
     build:
       context: ${PROJECT_ROOT}/packages/ddex/ingester
       dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile
+
+  ddex-test:
+    container_name: ddex-test
+    environment:
+      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file: .env
+    build:
+      context: ${PROJECT_ROOT}/packages/ddex/ingester
+      dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile.test
 
 networks:
   ddex-network:

dev-tools/compose/docker-compose.test.yml

@@ -448,7 +448,8 @@ services:
       service: ddex-crawler
     logging: *default-logging
     environment:
-      AWS_ENDPOINT: 'http://localstack:4566'
+      AWS_ENDPOINT: 'http://ddex-s3:4566'
+      TEST_MODE: 'true'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -460,6 +461,8 @@ services:
       file: docker-compose.yml
       service: ddex-indexer
     logging: *default-logging
+    environment:
+      AWS_ENDPOINT: 'http://ddex-s3:4566'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -471,6 +474,8 @@ services:
       file: docker-compose.yml
       service: ddex-parser
     logging: *default-logging
+    environment:
+      AWS_ENDPOINT: 'http://ddex-s3:4566'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -483,7 +488,7 @@ services:
       service: ddex-publisher
     logging: *default-logging
     environment:
-      AWS_ENDPOINT: 'http://localstack:4566'
+      AWS_ENDPOINT: 'http://ddex-s3:4566'
     depends_on:
       ddex-mongo-init:
         condition: service_completed_successfully
@@ -505,24 +510,29 @@ services:
       ddex-mongo:
         condition: service_healthy
 
-  # TODO: this will need to create the raw and indexed buckets
   # It also wouldn't be a bad idea to set CORS and use localstack in the local dev setup (ie audius-compose up --ddex-only)
   ddex-s3:
     container_name: ddex-s3
     image: localstack/localstack:s3-latest
     ports:
       - "127.0.0.1:4566:4566"
+    networks:
+      - ddex-network
     volumes:
       - "ddex-s3:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
 
   test-ddex-e2e:
     container_name: test-ddex-e2e
     extends:
-      file: docker-compose.yml
-      service: ddex-ingester
-    entrypoint: sh -c '[ ! "$$1" = "test" ] && sleep inf || (shift;  ./ingester --service e2e-test)' -
+      file: docker-compose.ddex.yml
+      service: ddex-test
+    entrypoint: sh -c '[ ! "$$1" = "test" ] && sleep inf || (shift;  go test ./e2e_test/... -count 1 -timeout 3m "$$@")' -
     logging: *default-logging
+    environment:
+      AWS_ENDPOINT: 'http://ddex-s3:4566'
+    networks:
+      - ddex-network
     depends_on:
       ddex-crawler:
         condition: service_healthy
@@ -533,14 +543,14 @@ services:
       # ddex-publisher:
       #   condition: service_healthy
       # Leaving out publisher for now because it takes a long time to build.
-      # We probably won't actually upload anything to Audius in the e2e test, but having a "dry run" publisher mode could be useful
+      # We don't actually upload anything to Audius in the e2e test, but having a "dry run" publisher mode could be useful
 
   test-ddex-unittests:
     container_name: test-ddex-unittests
-    build:
-      context: ${PROJECT_ROOT}/packages/ddex/ingester
-      dockerfile: Dockerfile.test
-    entrypoint: sh -c '[ ! "$$1" = "test" ] && sleep inf || (shift;  go test ./... -count 1 -timeout 60s "$$@")' -
+    extends:
+      file: docker-compose.ddex.yml
+      service: ddex-test
+    entrypoint: sh -c '[ ! "$$1" = "test" ] && sleep inf || (shift;  go test $(go list ./... | grep -v /e2e_test) -count 1 -timeout 60s "$$@")' -
     logging: *default-logging
     # Will need a depends_on for S3 and Mongo if any future unit tests use them
 

dev-tools/config.json

@@ -131,6 +131,9 @@
     "ETH_OWNER_WALLET": "0x855FA758c77D68a04990E992aA4dcdeF899F654A",
     "AWS_ACCESS_KEY_ID": "test",
     "AWS_SECRET_ACCESS_KEY": "test",
+    "AWS_REGION": "us-west-2",
+    "AWS_BUCKET_RAW": "audius-test-raw",
+    "AWS_BUCKET_INDEXED": "audius-test-indexed",
     "DDEX_KEY": "49d5e13d355709b615b7cce7369174fb240b6b39",
     "DDEX_SECRET": "2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152",
     "SESSION_SECRET": "something random"

packages/ddex/README.md

@@ -60,3 +60,7 @@ Each service can be run independently as long as `ddex-mongo` is up (from `audiu
     }
 ]
 ```
+
+### Running / debugging the e2e test
+* Run `audius-compose test down && audius-compose test run ddex-e2e` to start the ddex stack and run the e2e test
+* To debug S3, exec into `ddex-s3`, run `pip install awscli`, and then you can run `aws --endpoint=http://localhost:4566 s3 ls` and other commands to debug the S3 state

packages/ddex/ingester/cmd/main.go

@@ -50,10 +50,6 @@ func main() {
 		go indexer.RunNewIndexer(ctx)
 	case "parser":
 		go parser.RunNewParser(ctx)
-	case "e2e-test":
-		// TODO: Something like RunNewE2ETest(ctx)
-		fmt.Println("e2e-test not implemented")
-		cancel()
 	default:
 		fmt.Println("Unknown service: " + *service)
 	}

packages/ddex/ingester/common/common.go

@@ -157,10 +157,10 @@ func InitS3Client(logger *slog.Logger) (*s3.S3, *session.Session) {
 func MustGetenv(key string) string {
 	val := os.Getenv(key)
 	if val == "" {
-		log.Println("Missing required env variable: ", key, " sleeping ...")
+		log.Println("Missing required env variable:", key, "sleeping...")
 		// If config is incorrect, sleep a bit to prevent container from restarting constantly
 		time.Sleep(time.Hour)
-		log.Fatal("Missing required env variable: ", key)
+		log.Fatal("Missing required env variable:", key)
 	}
 	return val
 }

packages/ddex/ingester/crawler/crawler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"ingester/common"
+	"os"
 	"strings"
 	"time"
 
@@ -24,7 +25,11 @@ func RunNewCrawler(ctx context.Context) {
 	}
 	defer c.MongoClient.Disconnect(ctx)
 
-	ticker := time.NewTicker(3 * time.Minute)
+	interval := 3 * time.Minute
+	if os.Getenv("TEST_MODE") == "true" {
+		interval = time.Second
+	}
+	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
 	for {