From de7013540d0563e1c6baf6fbf5211f972a150b57 Mon Sep 17 00:00:00 2001
From: Theo Ilie <theoilie.ti@gmail.com>
Date: Tue, 6 Feb 2024 21:27:44 -0800
Subject: [PATCH] [PROTO-1621] Implement ddex indexer (#7486)

---
 packages/ddex/README.md                       |   4 +-
 packages/ddex/ingester/cmd/main.go            |   6 +-
 packages/ddex/ingester/common/common.go       |   4 +-
 packages/ddex/ingester/crawler/crawler.go     |   6 +-
 packages/ddex/ingester/go.mod                 |  31 +--
 packages/ddex/ingester/go.sum                 | 134 +--------
 packages/ddex/ingester/indexer/indexer.go     | 256 ++++++++++++++++--
 packages/ddex/ingester/parser/parser.go       |   9 +-
 packages/ddex/webapp/client/.env.dev          |   2 +-
 .../webapp/client/src/pages/Upload/Upload.tsx |   2 +-
 .../client/src/providers/EnvVarsProvider.tsx  |   2 +-
 packages/ddex/webapp/server/src/app.ts        |  31 ++-
 12 files changed, 292 insertions(+), 195 deletions(-)

diff --git a/packages/ddex/README.md b/packages/ddex/README.md
index 64df2aa076d..3bc7bc22902 100644
--- a/packages/ddex/README.md
+++ b/packages/ddex/README.md
@@ -22,7 +22,7 @@ chmod 400 packages/ddex/mongo-keyfile
 ```
 2. `audius-compose connect` to update your /etc/hosts
 3. `audius-compose up --ddex`
-4. Once the `ddex-mongo` container is running: manually intiate the mongodb replica set with `docker exec -it ddex-mongo mongosh -u mongo -p mongo --authenticationDatabase admin --eval 'rs.initiate({_id:"rs0", members:[{_id:0, host:"ddex-mongo:27017"}]})`. The other ddex containers will be blocked from starting until this command succeeds.
+4. Once the `ddex-mongo` container is running: manually intiate the mongodb replica set with `docker exec -it ddex-mongo mongosh -u mongo -p mongo --authenticationDatabase admin --eval 'rs.initiate({_id:"rs0", members:[{_id:0, host:"ddex-mongo:27017"}]})'`. The other ddex containers will be blocked from starting until this command succeeds.
 
 ### Bring up the ddex stack subsequently
 `audius-compose up --ddex`
@@ -32,7 +32,7 @@ Note: `audius-compose down` removes the `ddex-mongo-db` volume, so if you run th
 To access the ddex db via the mongo shell: `docker exec -it ddex-mongo mongosh -u mongo -p mongo --authenticationDatabase admin` then `use ddex`
 
 ### Develop with hot reloading
-Each service can be run independently as long as `ddex-mongo` is up. See the respective subdirectories' READMEs.
+Each service can be run independently as long as `ddex-mongo` is up (from `audius-compose up --ddex`` or `docker compose --project-directory=dev-tools/compose --file=dev-tools/compose/docker-compose.yml --profile=ddex up ddex-mongo -d`). See the respective subdirectories' READMEs.
 
 ### Creating a bucket in S3
 1. Create a new bucket in the S3 console with the name `ddex-[dev|staging]-<label/distributor>-raw`. Use all the defaults, including "ACLs disabled"
diff --git a/packages/ddex/ingester/cmd/main.go b/packages/ddex/ingester/cmd/main.go
index b51ed4b5ce7..948e1ebce0e 100644
--- a/packages/ddex/ingester/cmd/main.go
+++ b/packages/ddex/ingester/cmd/main.go
@@ -8,6 +8,7 @@ import (
 	"ingester/indexer"
 	"ingester/parser"
 	"log"
+	"log/slog"
 	"os"
 	"os/signal"
 	"syscall"
@@ -30,6 +31,9 @@ func main() {
 		cancel()
 	}()
 
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true}))
+	slog.SetDefault(logger)
+
 	err := godotenv.Load("../.env")
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -43,7 +47,7 @@ func main() {
 	case "crawler":
 		go crawler.Run(ctx)
 	case "indexer":
-		go indexer.Run(ctx)
+		go indexer.RunNewIndexer(ctx)
 	case "parser":
 		go parser.Run(ctx)
 	default:
diff --git a/packages/ddex/ingester/common/common.go b/packages/ddex/ingester/common/common.go
index 7b1c9c3f8b3..03fd742b1bb 100644
--- a/packages/ddex/ingester/common/common.go
+++ b/packages/ddex/ingester/common/common.go
@@ -27,7 +27,7 @@ func InitMongoClient(ctx context.Context) *mongo.Client {
 	return client
 }
 
-func InitS3Client() *s3.S3 {
+func InitS3Client() (*s3.S3, *session.Session) {
 	awsRegion := MustGetenv("AWS_REGION")
 	awsKey := MustGetenv("AWS_ACCESS_KEY_ID")
 	awsSecret := MustGetenv("AWS_SECRET_ACCESS_KEY")
@@ -39,7 +39,7 @@ func InitS3Client() *s3.S3 {
 		panic(err)
 	}
 	log.Println("Connected to s3")
-	return s3.New(sess)
+	return s3.New(sess), sess
 }
 
 func MustGetenv(key string) string {
diff --git a/packages/ddex/ingester/crawler/crawler.go b/packages/ddex/ingester/crawler/crawler.go
index 3739e1b3fa5..9720da7ab4f 100644
--- a/packages/ddex/ingester/crawler/crawler.go
+++ b/packages/ddex/ingester/crawler/crawler.go
@@ -15,7 +15,7 @@ import (
 )
 
 func Run(ctx context.Context) {
-	s3Client := common.InitS3Client()
+	s3Client, _ := common.InitS3Client()
 	mongoClient := common.InitMongoClient(ctx)
 	defer mongoClient.Disconnect(ctx)
 
@@ -115,8 +115,8 @@ func persistUploads(client *mongo.Client, bucket string, uploads []*s3.Object, c
 		path := "s3://" + bucket + "/" + *upload.Key
 		etag := strings.Trim(*upload.ETag, "\"")
 		// Only insert if a document doesn't already exist with this path and etag
-		filter := bson.M{"path": path, "delivery_etag": etag}
-		update := bson.M{"$setOnInsert": bson.M{"path": path, "delivery_etag": etag}}
+		filter := bson.M{"path": path, "upload_etag": etag}
+		update := bson.M{"$setOnInsert": bson.M{"path": path, "upload_etag": etag}}
 		opts := options.Update().SetUpsert(true)
 		_, err := uploadsColl.UpdateOne(ctx, filter, update, opts)
 		if err != nil {
diff --git a/packages/ddex/ingester/go.mod b/packages/ddex/ingester/go.mod
index a312b366a97..242c3db8b62 100644
--- a/packages/ddex/ingester/go.mod
+++ b/packages/ddex/ingester/go.mod
@@ -3,36 +3,25 @@ module ingester
 go 1.21.5
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
-	github.com/aws/aws-sdk-go v1.50.11 // indirect
-	github.com/bep/godartsass v1.2.0 // indirect
-	github.com/bep/godartsass/v2 v2.0.0 // indirect
-	github.com/bep/golibsass v1.1.1 // indirect
-	github.com/cli/safeexec v1.0.1 // indirect
-	github.com/cosmtrek/air v1.49.0 // indirect
-	github.com/creack/pty v1.1.21 // indirect
-	github.com/fatih/color v1.16.0 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/gohugoio/hugo v0.122.0 // indirect
+	github.com/aws/aws-sdk-go v1.50.11
+	github.com/joho/godotenv v1.5.1
+	go.mongodb.org/mongo-driver v1.13.1
+)
+
+require (
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/tdewolff/parse/v2 v2.7.11 // indirect
+	github.com/oklog/ulid/v2 v2.1.0
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.13.1 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
diff --git a/packages/ddex/ingester/go.sum b/packages/ddex/ingester/go.sum
index 126ad9acc8f..c8a8190a750 100644
--- a/packages/ddex/ingester/go.sum
+++ b/packages/ddex/ingester/go.sum
@@ -1,95 +1,29 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
-dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aws/aws-sdk-go v1.50.11 h1:BxUqMbkzKzzQ6FXLlZFNFkFjHnv6utbc4PItVs5SvaE=
 github.com/aws/aws-sdk-go v1.50.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
-github.com/bep/godartsass v1.2.0 h1:E2VvQrxAHAFwbjyOIExAMmogTItSKodoKuijNrGm5yU=
-github.com/bep/godartsass v1.2.0/go.mod h1:6LvK9RftsXMxGfsA0LDV12AGc4Jylnu6NgHL+Q5/pE8=
-github.com/bep/godartsass/v2 v2.0.0 h1:Ruht+BpBWkpmW+yAM2dkp7RSSeN0VLaTobyW0CiSP3Y=
-github.com/bep/godartsass/v2 v2.0.0/go.mod h1:AcP8QgC+OwOXEq6im0WgDRYK7scDsmZCEW62o1prQLo=
-github.com/bep/golibsass v1.1.1 h1:xkaet75ygImMYjM+FnHIT3xJn7H0xBA9UxSOJjk8Khw=
-github.com/bep/golibsass v1.1.1/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q=
-github.com/cli/safeexec v1.0.1 h1:e/C79PbXF4yYTN/wauC4tviMxEV13BwljGj0N9j+N00=
-github.com/cli/safeexec v1.0.1/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cosmtrek/air v1.49.0 h1:/LfwnLquRoRyUO/K+m/6N5YpUkxcKcDGq2dH7RYbT9c=
-github.com/cosmtrek/air v1.49.0/go.mod h1:yOz9vy7edZ75KRN9+Ofqwm3OU0wuv4Csc+ikMeZxxS8=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
-github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
-github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/gohugoio/hugo v0.122.0 h1:zsmYxRSxYjn8GxXUYCrQtZgJ6pYlsUoaQn0BSmI76fw=
-github.com/gohugoio/hugo v0.122.0/go.mod h1:jvoQfOvoyftwdqxRsXNYkdyVROtViJ9GDzK4SLxOreE=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
-github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
+github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
-github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tdewolff/parse/v2 v2.7.11 h1:v+W45LnzmjndVlfqPCT5gGjAAZKd1GJGOPJveTIkBY8=
-github.com/tdewolff/parse/v2 v2.7.11/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
-github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
@@ -103,42 +37,26 @@ go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -146,43 +64,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/packages/ddex/ingester/indexer/indexer.go b/packages/ddex/ingester/indexer/indexer.go
index 7f996f76453..a0d2d46ffdb 100644
--- a/packages/ddex/ingester/indexer/indexer.go
+++ b/packages/ddex/ingester/indexer/indexer.go
@@ -1,35 +1,66 @@
 package indexer
 
 import (
+	"archive/zip"
 	"context"
+	"fmt"
 	"ingester/common"
-	"ingester/constants"
+	"io"
 	"log"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/oklog/ulid/v2"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
-func Run(ctx context.Context) {
+type Indexer struct {
+	s3Downloader  *s3manager.Downloader
+	s3Uploader    *s3manager.Uploader
+	mongoClient   *mongo.Client
+	rawBucket     string
+	indexedBucket string
+	indexedColl   *mongo.Collection
+	ctx           context.Context
+	logger        *slog.Logger
+}
+
+// RunNewIndexer starts the indexer service, which listens for new uploads in the Mongo "uploads" collection and processes them.
+func RunNewIndexer(ctx context.Context) {
+	_, s3Session := common.InitS3Client()
 	mongoClient := common.InitMongoClient(ctx)
 	defer mongoClient.Disconnect(ctx)
+	indexedColl := mongoClient.Database("ddex").Collection("indexed")
+
+	i := &Indexer{
+		s3Downloader:  s3manager.NewDownloader(s3Session),
+		s3Uploader:    s3manager.NewUploader(s3Session),
+		mongoClient:   mongoClient,
+		rawBucket:     common.MustGetenv("AWS_BUCKET_RAW"),
+		indexedBucket: common.MustGetenv("AWS_BUCKET_INDEXED"),
+		indexedColl:   indexedColl,
+		ctx:           ctx,
+		logger:        slog.With("service", "indexer"),
+	}
 
 	uploadsColl := mongoClient.Database("ddex").Collection("uploads")
-	pipeline := mongo.Pipeline{bson.D{{"$match", bson.D{{"operationType", "insert"}}}}}
+	pipeline := mongo.Pipeline{bson.D{{Key: "$match", Value: bson.D{{Key: "operationType", Value: "insert"}}}}}
 	changeStream, err := uploadsColl.Watch(ctx, pipeline)
 	if err != nil {
 		panic(err)
 	}
-	log.Println("Watching collection 'uploads'")
+	i.logger.Info("Indexer: Watching collection 'uploads'")
 	defer changeStream.Close(ctx)
 
 	for changeStream.Next(ctx) {
-		var changeDoc bson.M
-		if err := changeStream.Decode(&changeDoc); err != nil {
-			log.Fatal(err)
-		}
-		fullDocument, _ := changeDoc["fullDocument"].(bson.M)
-		indexUpload(mongoClient, fullDocument, ctx)
+		i.processZIP(changeStream)
 	}
 
 	if err := changeStream.Err(); err != nil {
@@ -37,26 +68,201 @@ func Run(ctx context.Context) {
 	}
 }
 
-func indexUpload(client *mongo.Client, fullDocument bson.M, ctx context.Context) {
-	log.Printf("Processing new upload: %v\n", fullDocument)
-	indexedColl := client.Database("ddex").Collection("indexed")
+// processZIP unzips an "upload" into a "delivery" (or multiple deliveries if the ZIP file contains multiple folders with XML files).
+func (i *Indexer) processZIP(changeStream *mongo.ChangeStream) {
+	// Decode the "upload" from Mongo
+	var changeDoc bson.M
+	if err := changeStream.Decode(&changeDoc); err != nil {
+		log.Fatal(err)
+	}
+	fullDocument, _ := changeDoc["fullDocument"].(bson.M)
+	i.logger.Info("Indexer: Processing new upload", "upload", fullDocument)
+
+	// Download ZIP file from S3
+	uploadETag := fullDocument["upload_etag"].(string)
+	remotePath := fullDocument["path"].(string)
+	zipFilePath, cleanup := i.downloadFromS3Raw(remotePath)
+	defer cleanup()
+	if zipFilePath == "" {
+		i.logger.Error("Failed to download ZIP file from S3 bucket", "path", remotePath)
+		return
+	}
+
+	// Unzip the file and process its contents
+	extractDir, err := os.MkdirTemp("", "extracted")
+	if err != nil {
+		i.logger.Error("Error creating temp directory", "error", err)
+		return
+	}
+	defer os.RemoveAll(extractDir)
+
+	if err := unzip(zipFilePath, extractDir); err != nil {
+		i.logger.Error("Error unzipping file", "error", err)
+		return
+	}
 
-	delete(fullDocument, "_id")
+	i.processZIPContents(extractDir, uploadETag)
+}
 
-	// TODO process upload:
-	// 1. download zip from raw s3 bucket
-	// 2. unzip
-	// 3. upload files to indexed s3 bucket
+// processZIPContents finds deliveries in rootDir (and its subdirectories), uploads their contents to S3, and inserts them into the MongoDB "indexed" collection.
+func (i *Indexer) processZIPContents(rootDir, uploadETag string) error {
+	// The root directory could be the single delivery
+	i.processDelivery(rootDir, rootDir, uploadETag)
 
-	// Write delivery to 'indexed' collection
-	fullDocument["delivery_status"] = constants.DeliveryStatusValidating
-	// TODO download xml from bucket
-	// fullDocument["delivery_xml"] = ...
+	// Or it could contain multiple deliveries
+	return filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if strings.Contains(path, "__MACOSX") {
+			return nil
+		}
+		if info.IsDir() {
+			if path == rootDir {
+				return nil
+			}
+			return i.processDelivery(rootDir, path, uploadETag)
+		}
+		return nil
+	})
+}
 
-	result, err := indexedColl.InsertOne(ctx, fullDocument)
+// processDelivery parses a "delivery" from dir if dir contains an XML file.
+func (i *Indexer) processDelivery(rootDir, dir, uploadETag string) error {
+	var deliveryID, xmlRelativePath string
+	var xmlBytes []byte
+	files, err := os.ReadDir(dir)
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("failed to read directory %s: %w", dir, err)
+	}
+
+	// Look for the XML first and make a new deliveryID for it
+	for _, file := range files {
+		if !file.IsDir() && strings.HasSuffix(file.Name(), ".xml") {
+			deliveryID = ulid.Make().String()
+			xmlPath := filepath.Join(dir, file.Name())
+			xmlBytes, err = os.ReadFile(xmlPath)
+			if err != nil {
+				return fmt.Errorf("failed to read XML file %s: %w", xmlPath, err)
+			}
+
+			xmlRelativePath, err = filepath.Rel(rootDir, xmlPath)
+			if err != nil {
+				return fmt.Errorf("failed to compute relative path for %s: %w", xmlPath, err)
+			}
+			break // Assume only one XML file per directory
+		}
+	}
+
+	if deliveryID == "" {
+		i.logger.Info("No XML file found in directory. Skipping", "dir", dir)
+		return nil
+	}
+
+	// Upload the delivery's audio and images to S3
+	for _, file := range files {
+		if !file.IsDir() && !strings.HasSuffix(file.Name(), ".xml") {
+			filePath := filepath.Join(dir, file.Name())
+			i.uploadToS3Indexed(filePath, fmt.Sprintf("%s/%s", deliveryID, file.Name()))
+		}
+	}
+
+	// Insert the delivery into the Mongo "indexed" collection
+	deliveryDoc := bson.M{
+		"delivery_id":   deliveryID,
+		"upload_etag":   uploadETag,
+		"xml_file_path": xmlRelativePath,
+		"xml_content":   primitive.Binary{Data: xmlBytes, Subtype: 0x00}, // Store directly as generic binary for high data integrity
+	}
+	if _, err := i.indexedColl.InsertOne(i.ctx, deliveryDoc); err != nil {
+		return fmt.Errorf("failed to insert XML data into Mongo: %w", err)
+	}
+
+	return nil
+}
+
+// downloadFromRaw downloads a file from the S3 "raw" bucket to a temporary file.
+func (i *Indexer) downloadFromS3Raw(remotePath string) (string, func()) {
+	if !strings.HasPrefix(remotePath, "s3://"+i.rawBucket+"/") {
+		i.logger.Error("Invalid S3 path", "path", remotePath)
+		return "", func() {}
+	}
+	s3Key := strings.TrimPrefix(remotePath, "s3://"+i.rawBucket+"/")
+	file, err := os.CreateTemp("", "*.zip")
+	if err != nil {
+		i.logger.Error("Error creating temp file", "error", err)
+		return "", func() {}
+	}
+
+	_, err = i.s3Downloader.Download(file, &s3.GetObjectInput{
+		Bucket: aws.String(i.rawBucket),
+		Key:    aws.String(s3Key),
+	})
+	if err != nil {
+		i.logger.Error("Error downloading file from S3", "error", err)
+		file.Close()
+		os.Remove(file.Name())
+		return "", func() {}
+	}
+	file.Close()
+	return file.Name(), func() { os.Remove(file.Name()) }
+}
+
+// uploadToIndexed uploads a file to the S3 "indexed" bucket.
+func (i *Indexer) uploadToS3Indexed(filePath, fileKey string) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		i.logger.Error("Error opening file", "error", err)
+		return
+	}
+	defer file.Close()
+
+	_, err = i.s3Uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(i.indexedBucket),
+		Key:    aws.String(fileKey),
+		Body:   file,
+	})
+	if err != nil {
+		i.logger.Error("Error uploading file to S3", "error", err)
+	} else {
+		i.logger.Info("Uploaded file to S3", "file", filePath, "key", fileKey)
+	}
+}
+
+// unzip extracts the contents of the zip file at src to the directory at dest.
+func unzip(src, dest string) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
 	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		fpath := filepath.Join(dest, f.Name)
+		if f.FileInfo().IsDir() {
+			os.MkdirAll(fpath, os.ModePerm)
+		} else {
+			if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+				return err
+			}
+			outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			if err != nil {
+				return err
+			}
+			rc, err := f.Open()
+			if err != nil {
+				return err
+			}
+			_, err = io.Copy(outFile, rc)
 
-	log.Println("New indexed doc ID: ", result.InsertedID)
+			// Close the file without deferring to ensure we don't open too many files
+			outFile.Close()
+			rc.Close()
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
diff --git a/packages/ddex/ingester/parser/parser.go b/packages/ddex/ingester/parser/parser.go
index 45db16b6dc5..bde10322f6c 100644
--- a/packages/ddex/ingester/parser/parser.go
+++ b/packages/ddex/ingester/parser/parser.go
@@ -17,7 +17,7 @@ func Run(ctx context.Context) {
 	defer mongoClient.Disconnect(ctx)
 
 	indexedColl := mongoClient.Database("ddex").Collection("indexed")
-	pipeline := mongo.Pipeline{bson.D{{"$match", bson.D{{"operationType", "insert"}}}}}
+	pipeline := mongo.Pipeline{bson.D{{Key: "$match", Value: bson.D{{Key: "operationType", Value: "insert"}}}}}
 	changeStream, err := indexedColl.Watch(ctx, pipeline)
 	if err != nil {
 		panic(err)
@@ -58,9 +58,10 @@ func parseIndexed(client *mongo.Client, fullDocument bson.M, ctx context.Context
 
 		// 2. Write each release in "delivery_xml" in the indexed doc as a bson doc in the 'parsed' collection
 		parsedDoc := bson.M{
-			"delivery_etag": fullDocument["delivery_etag"],
-			"entity":        "track",
-			"publish_date":  time.Now(),
+			"upload_etag":  fullDocument["upload_etag"],
+			"delivery_id":  fullDocument["delivery_id"],
+			"entity":       "track",
+			"publish_date": time.Now(),
 		}
 		result, err := parsedColl.InsertOne(ctx, parsedDoc)
 		if err != nil {
diff --git a/packages/ddex/webapp/client/.env.dev b/packages/ddex/webapp/client/.env.dev
index 29ead550fd5..c9b88a4c0d7 100644
--- a/packages/ddex/webapp/client/.env.dev
+++ b/packages/ddex/webapp/client/.env.dev
@@ -1,4 +1,4 @@
 # For testing staging locally (ie, npm run stage)
 VITE_ENV_OVERRIDE=sandbox
-VITE_NODE_URL_OVERRIDE=https://localhost:9000
+VITE_NODE_URL_OVERRIDE=http://localhost:9000
 VITE_OPTIMIZELY_SDK_KEY_OVERRIDE=MX4fYBgANQetvmBXGpuxzF
diff --git a/packages/ddex/webapp/client/src/pages/Upload/Upload.tsx b/packages/ddex/webapp/client/src/pages/Upload/Upload.tsx
index 026e7766ba2..0bd48cce314 100644
--- a/packages/ddex/webapp/client/src/pages/Upload/Upload.tsx
+++ b/packages/ddex/webapp/client/src/pages/Upload/Upload.tsx
@@ -14,7 +14,7 @@ import { trpc } from 'utils/trpc'
 
 import styles from './Upload.module.css'
 
-const MAX_SIZE = 5 * 1024 * 1024 // 5 MB
+const MAX_SIZE = 500 * 1024 * 1024 // 500 MB
 
 const validZipFile = (file: File) => {
   if (
diff --git a/packages/ddex/webapp/client/src/providers/EnvVarsProvider.tsx b/packages/ddex/webapp/client/src/providers/EnvVarsProvider.tsx
index ddafb20f80a..30a1d81a2b5 100644
--- a/packages/ddex/webapp/client/src/providers/EnvVarsProvider.tsx
+++ b/packages/ddex/webapp/client/src/providers/EnvVarsProvider.tsx
@@ -46,7 +46,7 @@ export const EnvVarsProvider = ({ children }: { children: ReactNode }) => {
     }
 
     // Stage and prod nodes fetch env vars exposed in audius-docker-compose
-    if (!envVars.env) {
+    if (!envVars?.ddexKey) {
       fetchEnvVars()
     }
   })
diff --git a/packages/ddex/webapp/server/src/app.ts b/packages/ddex/webapp/server/src/app.ts
index 34b870e9e82..72be1039301 100644
--- a/packages/ddex/webapp/server/src/app.ts
+++ b/packages/ddex/webapp/server/src/app.ts
@@ -1,4 +1,5 @@
 import express, { Express, Request, Response } from 'express'
+import cors from 'cors'
 import path from 'path'
 
 export default function createApp() {
@@ -11,19 +12,25 @@ export default function createApp() {
    * Define API routes
    */
 
-  app.get('/api/env', (_req: Request, res: Response) => {
-    const envData = {
-      data: {
-        env: process.env.NODE_ENV,
-        awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
-        awsBucketRaw: process.env.AWS_BUCKET_RAW,
-        awsBucketIndexed: process.env.AWS_BUCKET_INDEXED,
-        ddexKey: process.env.DDEX_KEY,
-        optimizelySdkKey: process.env.OPTIMIZELY_SDK_KEY,
-      },
+  app.get(
+    '/api/env',
+    cors({
+      origin: 'http://localhost:5173',
+    }),
+    (_req: Request, res: Response) => {
+      const envData = {
+        data: {
+          env: process.env.NODE_ENV,
+          awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
+          awsBucketRaw: process.env.AWS_BUCKET_RAW,
+          awsBucketIndexed: process.env.AWS_BUCKET_INDEXED,
+          ddexKey: process.env.DDEX_KEY,
+          optimizelySdkKey: process.env.OPTIMIZELY_SDK_KEY,
+        },
+      }
+      res.json(envData)
     }
-    res.json(envData)
-  })
+  )
   app.get('/api/health_check', (_req: Request, res: Response) => {
     res.status(200).send('DDEX is alive!')
   })