feat: add batchsize to config

penguin-statistics · Nov 21, 2023 · ac61e20 · ac61e20
1 parent d5aae82
commit ac61e20
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/internal/app/appconfig/spec.go b/internal/app/appconfig/spec.go
@@ -131,6 +131,8 @@ type ConfigSpec struct {
 	// We don't want to show all patterns because it will be too many. So we set a limit here (default 19)
 	PatternMatrixLimit int `split_words:"true" default:"19"`
 
+	DropReportArchiveBatchSize int `split_words:"true" default:"1000"`
+
 	DropReportArchiveS3Bucket string `required:"true" split_words:"true"`
 	DropReportArchiveS3Region string `required:"true" split_words:"true"`
 	AWSAccessKey              string `required:"true" split_words:"true"`

diff --git a/internal/service/drop_report_archive.go b/internal/service/drop_report_archive.go
@@ -22,7 +22,6 @@ const (
 	RealmDropReportExtras = "drop_report_extras"
 
 	ArchiveS3Prefix = "v1/"
-	BatchSize       = 1000
 )
 
 type Archive struct {
@@ -118,7 +117,7 @@ func (s *Archive) populateDropReportsToArchiver(ctx context.Context, date time.T
 	var err error
 	var page, totalCount, firstId, lastId int
 	for {
-		dropReports, cursor, err = s.DropReportService.GetDropReportsForArchive(ctx, &cursor, date, 1000)
+		dropReports, cursor, err = s.DropReportService.GetDropReportsForArchive(ctx, &cursor, date, s.Config.DropReportArchiveBatchSize)
 		if err != nil {
 			return 0, 0, errors.Wrap(err, "failed to extract drop reports")
 		}
@@ -160,7 +159,7 @@ func (s *Archive) populateDropReportExtrasToArchiver(ctx context.Context, idIncl
 	var err error
 	var page, totalCount int
 	for {
-		extras, cursor, err = s.DropReportExtraService.GetDropReportExtraForArchive(ctx, &cursor, idInclusiveStart, idInclusiveEnd, 1000)
+		extras, cursor, err = s.DropReportExtraService.GetDropReportExtraForArchive(ctx, &cursor, idInclusiveStart, idInclusiveEnd, s.Config.DropReportArchiveBatchSize)
 		if err != nil {
 			return errors.Wrap(err, "failed to extract drop report extras")
 		}