From ac61e207e6704dd65ce46b2799fd981152b8fd6e Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Tue, 21 Nov 2023 14:19:46 -0800 Subject: [PATCH] feat: add batchsize to config --- internal/app/appconfig/spec.go | 2 ++ internal/service/drop_report_archive.go | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/app/appconfig/spec.go b/internal/app/appconfig/spec.go index a54d0e28..c6be7401 100644 --- a/internal/app/appconfig/spec.go +++ b/internal/app/appconfig/spec.go @@ -131,6 +131,8 @@ type ConfigSpec struct { // We don't want to show all patterns because it will be too many. So we set a limit here (default 19) PatternMatrixLimit int `split_words:"true" default:"19"` + DropReportArchiveBatchSize int `split_words:"true" default:"1000"` + DropReportArchiveS3Bucket string `required:"true" split_words:"true"` DropReportArchiveS3Region string `required:"true" split_words:"true"` AWSAccessKey string `required:"true" split_words:"true"` diff --git a/internal/service/drop_report_archive.go b/internal/service/drop_report_archive.go index 3e87de09..310b0fbb 100644 --- a/internal/service/drop_report_archive.go +++ b/internal/service/drop_report_archive.go @@ -22,7 +22,6 @@ const ( RealmDropReportExtras = "drop_report_extras" ArchiveS3Prefix = "v1/" - BatchSize = 1000 ) type Archive struct { @@ -118,7 +117,7 @@ func (s *Archive) populateDropReportsToArchiver(ctx context.Context, date time.T var err error var page, totalCount, firstId, lastId int for { - dropReports, cursor, err = s.DropReportService.GetDropReportsForArchive(ctx, &cursor, date, 1000) + dropReports, cursor, err = s.DropReportService.GetDropReportsForArchive(ctx, &cursor, date, s.Config.DropReportArchiveBatchSize) if err != nil { return 0, 0, errors.Wrap(err, "failed to extract drop reports") } @@ -160,7 +159,7 @@ func (s *Archive) populateDropReportExtrasToArchiver(ctx context.Context, idIncl var err error var page, totalCount int for { - extras, cursor, err = s.DropReportExtraService.GetDropReportExtraForArchive(ctx, &cursor, idInclusiveStart, idInclusiveEnd, 1000) + extras, cursor, err = s.DropReportExtraService.GetDropReportExtraForArchive(ctx, &cursor, idInclusiveStart, idInclusiveEnd, s.Config.DropReportArchiveBatchSize) if err != nil { return errors.Wrap(err, "failed to extract drop report extras") }