From 82f673ffead1eeaee08dec1461135d4bb151359d Mon Sep 17 00:00:00 2001
From: achettyiitr <achetty.iitr@gmail.com>
Date: Mon, 5 Aug 2024 19:37:01 +0530
Subject: [PATCH] chore: cleanup duplicate minio contents

---
 router/batchrouter/batchrouter_test.go | 46 ++++++++------------------
 warehouse/archive/archiver_test.go     | 37 ++++-----------------
 2 files changed, 20 insertions(+), 63 deletions(-)

diff --git a/router/batchrouter/batchrouter_test.go b/router/batchrouter/batchrouter_test.go
index fc119d2a1a..521e6efd01 100644
--- a/router/batchrouter/batchrouter_test.go
+++ b/router/batchrouter/batchrouter_test.go
@@ -1,7 +1,6 @@
 package batchrouter
 
 import (
-	"compress/gzip"
 	"context"
 	jsonb "encoding/json"
 	"errors"
@@ -15,7 +14,6 @@ import (
 	"testing"
 	"time"
 
-	miniogo "github.com/minio/minio-go/v7"
 	"github.com/ory/dockertest/v3"
 	"github.com/samber/lo"
 
@@ -36,6 +34,7 @@ import (
 	"github.com/rudderlabs/rudder-go-kit/logger"
 	"github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/minio"
 	"github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres"
+
 	"github.com/rudderlabs/rudder-server/admin"
 	backendconfig "github.com/rudderlabs/rudder-server/backend-config"
 	"github.com/rudderlabs/rudder-server/jobsdb"
@@ -652,39 +651,20 @@ func TestBatchRouter(t *testing.T) {
 	defer batchrouter.Shutdown()
 
 	require.Eventually(t, func() bool {
-		return len(minioContents(t, context.Background(), minioResource, "")) == len(bcs)
+		minioContents, err := minioResource.Contents(context.Background(), "")
+		if err != nil {
+			t.Logf("error getting minio contents: %v", err)
+			return false
+		}
+		return len(minioContents) == len(bcs)
 	}, 5*time.Second, 200*time.Millisecond)
 
-	filenames := lo.Map(
-		lo.Keys(minioContents(t, context.Background(), minioResource, "")),
-		func(k string, _ int) string { return path.Dir(k) },
-	)
-
-	require.ElementsMatch(t, filePrefixes, filenames)
-}
-
-func minioContents(t require.TestingT, ctx context.Context, dest *minio.Resource, prefix string) map[string]string {
-	contents := make(map[string]string)
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	opts := miniogo.ListObjectsOptions{
-		Recursive: true,
-		Prefix:    prefix,
-	}
-	for objInfo := range dest.Client.ListObjects(ctx, dest.BucketName, opts) {
-		o, err := dest.Client.GetObject(ctx, dest.BucketName, objInfo.Key, miniogo.GetObjectOptions{})
-		require.NoError(t, err)
-
-		g, err := gzip.NewReader(o)
-		require.NoError(t, err)
-
-		b, err := io.ReadAll(g)
-		require.NoError(t, err)
+	minioContents, err := minioResource.Contents(context.Background(), "")
+	require.NoError(t, err)
 
-		contents[objInfo.Key] = string(b)
-	}
+	filenames := lo.Map(minioContents, func(k minio.File, _ int) string {
+		return path.Dir(k.Key)
+	})
 
-	return contents
+	require.ElementsMatch(t, filePrefixes, filenames)
 }
diff --git a/warehouse/archive/archiver_test.go b/warehouse/archive/archiver_test.go
index 6b0e798c15..188e3d1b4f 100644
--- a/warehouse/archive/archiver_test.go
+++ b/warehouse/archive/archiver_test.go
@@ -1,19 +1,17 @@
 package archive_test
 
 import (
-	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
 
-	miniogo "github.com/minio/minio-go/v7"
 	"github.com/ory/dockertest/v3"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 	"golang.org/x/sync/errgroup"
@@ -186,7 +184,12 @@ func TestArchiver(t *testing.T) {
 				}
 			}
 
-			contents := minioContents(t, ctx, minioResource, prefix)
+			minioContents, err := minioResource.Contents(ctx, prefix)
+			require.NoError(t, err)
+
+			contents := lo.SliceToMap(minioContents, func(item minio.File) (string, string) {
+				return item.Key, item.Content
+			})
 
 			var expectedContents map[string]string
 			jsonTestData(t, "testdata/storage.json", &expectedContents)
@@ -211,32 +214,6 @@ func TestArchiver(t *testing.T) {
 	}
 }
 
-func minioContents(t require.TestingT, ctx context.Context, dest *minio.Resource, prefix string) map[string]string {
-	contents := make(map[string]string)
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	opts := miniogo.ListObjectsOptions{
-		Recursive: true,
-		Prefix:    prefix,
-	}
-	for objInfo := range dest.Client.ListObjects(ctx, dest.BucketName, opts) {
-		o, err := dest.Client.GetObject(ctx, dest.BucketName, objInfo.Key, miniogo.GetObjectOptions{})
-		require.NoError(t, err)
-
-		g, err := gzip.NewReader(o)
-		require.NoError(t, err)
-
-		b, err := io.ReadAll(g)
-		require.NoError(t, err)
-
-		contents[objInfo.Key] = string(b)
-	}
-
-	return contents
-}
-
 func jsonTestData(t require.TestingT, file string, value any) {
 	f, err := os.Open(file)
 	require.NoError(t, err)