From 07aba1c008c7bfe4e33efdf7b115dd90b161bc18 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 7 Nov 2024 07:42:02 -0800 Subject: [PATCH 1/8] add S3 minimum part sized defined by the user Signed-off-by: Renan Rangel --- go/vt/mysqlctl/s3backupstorage/s3.go | 32 +++++++++-- go/vt/mysqlctl/s3backupstorage/s3_test.go | 65 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index 97861e83729..344d0531223 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -47,6 +47,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" transport "github.com/aws/smithy-go/endpoints" "github.com/aws/smithy-go/middleware" + "github.com/dustin/go-humanize" "github.com/spf13/pflag" errorsbackup "vitess.io/vitess/go/vt/mysqlctl/errors" @@ -86,6 +87,11 @@ var ( // path component delimiter delimiter = "/" + + // minimum part size + minimumPartSize int64 + + ErrPartSize = errors.New("minimum S3 part size must be between 5MiB and 5GiB") ) func registerFlags(fs *pflag.FlagSet) { @@ -98,6 +104,7 @@ func registerFlags(fs *pflag.FlagSet) { fs.BoolVar(&tlsSkipVerifyCert, "s3_backup_tls_skip_verify_cert", false, "skip the 'certificate is valid' check for SSL connections.") fs.StringVar(&requiredLogLevel, "s3_backup_log_level", "LogOff", "determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors.") fs.StringVar(&sse, "s3_backup_server_side_encryption", "", "server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file).") + fs.Int64Var(&minimumPartSize, "s3_backup_aws_minimum_partsize", 1024*1024*5, "Minimum part size to use") } func init() { @@ -166,7 +173,12 @@ func (bh *S3BackupHandle) AddFile(ctx context.Context, filename string, filesize return nil, fmt.Errorf("AddFile cannot be called on read-only backup") } - partSizeBytes := calculateUploadPartSize(filesize) + partSizeBytes, err := calculateUploadPartSize(filesize) + if err != nil { + return nil, err + } + + bh.bs.params.Logger.Infof("Using S3 upload part size: %s", humanize.IBytes(uint64(partSizeBytes))) reader, writer := io.Pipe() bh.handleAddFile(ctx, filename, partSizeBytes, reader, func(err error) { @@ -213,9 +225,11 @@ func (bh *S3BackupHandle) handleAddFile(ctx context.Context, filename string, pa }() } -func calculateUploadPartSize(filesize int64) int64 { +// this is a helper to calculate the part size, taking into consideration the minimum part size +// passed in by an operator. +func calculateUploadPartSize(filesize int64) (partSizeBytes int64, err error) { // Calculate s3 upload part size using the source filesize - partSizeBytes := manager.DefaultUploadPartSize + partSizeBytes = manager.DefaultUploadPartSize if filesize > 0 { minimumPartSize := float64(filesize) / float64(manager.MaxUploadParts) // Round up to ensure large enough partsize @@ -224,7 +238,17 @@ func calculateUploadPartSize(filesize int64) int64 { partSizeBytes = calculatedPartSizeBytes } } - return partSizeBytes + + if minimumPartSize != 0 && partSizeBytes < minimumPartSize { + if minimumPartSize > 1024*1024*1024*5 || minimumPartSize < 1024*1024*5 { // 5GiB and 5MiB respectively + return 0, fmt.Errorf("%w, currently set to %s", + ErrPartSize, humanize.IBytes(uint64(minimumPartSize)), + ) + } + partSizeBytes = int64(minimumPartSize) + } + + return } // EndBackup is part of the backupstorage.BackupHandle interface. diff --git a/go/vt/mysqlctl/s3backupstorage/s3_test.go b/go/vt/mysqlctl/s3backupstorage/s3_test.go index 84ef8de6e48..033207bfdb0 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3_test.go +++ b/go/vt/mysqlctl/s3backupstorage/s3_test.go @@ -328,3 +328,68 @@ func TestWithParams(t *testing.T) { assert.NotNil(t, s3.transport.DialContext) assert.NotNil(t, s3.transport.Proxy) } + +func TestGetPartSize(t *testing.T) { + originalMinimum := minimumPartSize + defer func() { minimumPartSize = originalMinimum }() + + tests := []struct { + name string + filesize int64 + minimumPartSize int64 + want int64 + err error + }{ + { + name: "minimum - 10 MiB", + filesize: 1024 * 1024 * 10, // 10 MiB + minimumPartSize: 1024 * 1024 * 5, // 5 MiB + want: 1024 * 1024 * 5, // 5 MiB, + err: nil, + }, + { + name: "below minimum - 10 MiB", + filesize: 1024 * 1024 * 10, // 10 MiB + minimumPartSize: 1024 * 1024 * 8, // 8 MiB + want: 1024 * 1024 * 8, // 8 MiB, + err: nil, + }, + { + name: "above minimum - 1 TiB", + filesize: 1024 * 1024 * 1024 * 1024, // 1 TiB + minimumPartSize: 1024 * 1024 * 5, // 5 MiB + want: 109951163, // ~104 MiB + err: nil, + }, + { + name: "below minimum - 1 TiB", + filesize: 1024 * 1024 * 1024 * 1024, // 1 TiB + minimumPartSize: 1024 * 1024 * 200, // 200 MiB + want: 1024 * 1024 * 200, // 200 MiB + err: nil, + }, + { + name: "below S3 limits - 5 MiB", + filesize: 1024 * 1024 * 3, // 3 MiB + minimumPartSize: 1024 * 1024 * 4, // 4 MiB + want: 1024 * 1024 * 5, // 5 MiB - should always return the minimum + err: nil, + }, + { + name: "above S3 limits - 5 GiB", + filesize: 1024 * 1024 * 1024 * 1024, // 1 TiB + minimumPartSize: 1024 * 1024 * 1024 * 6, // 6 GiB + want: 0, + err: ErrPartSize, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + minimumPartSize = tt.minimumPartSize + partSize, err := getPartSize(tt.filesize) + require.ErrorIs(t, err, tt.err) + require.Equal(t, tt.want, partSize) + }) + } +} From 5c4694b1058b4d75a3517fc8e8eb1a29868ca6e9 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 7 Nov 2024 08:07:45 -0800 Subject: [PATCH 2/8] update vttablet.txt Signed-off-by: Renan Rangel --- go/flags/endtoend/vttablet.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index e4c6fde66af..be5917f789b 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -312,6 +312,7 @@ Flags: --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). + --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. From c4a59882af1e18adba5cd2abd2f610aaa58bdc1f Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 7 Nov 2024 11:12:49 -0800 Subject: [PATCH 3/8] fixing flags Signed-off-by: Renan Rangel --- go/flags/endtoend/vtbackup.txt | 1 + go/flags/endtoend/vtctld.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index bf3a9eb9690..31842910205 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -195,6 +195,7 @@ Flags: --remote_operation_timeout duration time to wait for a remote operation (default 15s) --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). + --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 8b1aa6f4a92..45235d95a48 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -110,6 +110,7 @@ Flags: --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) --remote_operation_timeout duration time to wait for a remote operation (default 15s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). + --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. From 9ab638af144f9256ad0c9246bfc86ba618d545a2 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 21 Nov 2024 06:33:30 -0800 Subject: [PATCH 4/8] PR feedback Signed-off-by: Renan Rangel --- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/vt/mysqlctl/s3backupstorage/s3.go | 19 +++++++++++-------- go/vt/mysqlctl/s3backupstorage/s3_mock.go | 12 ++++++++++-- go/vt/mysqlctl/s3backupstorage/s3_test.go | 10 +++++----- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 31842910205..650a035b6fe 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -195,7 +195,7 @@ Flags: --remote_operation_timeout duration time to wait for a remote operation (default 15s) --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). - --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) + --s3_backup_aws_min_partsize int Minimum part size to use, defaults to 5MiB but can be increased due to the dataset size. (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 45235d95a48..bd51d594215 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -110,7 +110,7 @@ Flags: --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) --remote_operation_timeout duration time to wait for a remote operation (default 15s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). - --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) + --s3_backup_aws_min_partsize int Minimum part size to use, defaults to 5MiB but can be increased due to the dataset size. (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index be5917f789b..e26bea8d65f 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -312,7 +312,7 @@ Flags: --restore_from_backup_ts string (init restore parameter) if set, restore the latest backup taken at or before this timestamp. Example: '2021-04-29.133050' --retain_online_ddl_tables duration How long should vttablet keep an old migrated table before purging it (default 24h0m0s) --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). - --s3_backup_aws_minimum_partsize int Minimum part size to use (default 5242880) + --s3_backup_aws_min_partsize int Minimum part size to use, defaults to 5MiB but can be increased due to the dataset size. (default 5242880) --s3_backup_aws_region string AWS region to use. (default "us-east-1") --s3_backup_aws_retries int AWS request retries. (default -1) --s3_backup_force_path_style force the s3 path style. diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index 344d0531223..c759108583b 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -58,6 +58,11 @@ import ( "vitess.io/vitess/go/vt/servenv" ) +const ( + sseCustomerPrefix = "sse_c:" + MaxPartSize = 1024 * 1024 * 1024 * 5 // 5GiB - limited by AWS https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html +) + var ( // AWS API region region string @@ -89,7 +94,7 @@ var ( delimiter = "/" // minimum part size - minimumPartSize int64 + minPartSize int64 ErrPartSize = errors.New("minimum S3 part size must be between 5MiB and 5GiB") ) @@ -104,7 +109,7 @@ func registerFlags(fs *pflag.FlagSet) { fs.BoolVar(&tlsSkipVerifyCert, "s3_backup_tls_skip_verify_cert", false, "skip the 'certificate is valid' check for SSL connections.") fs.StringVar(&requiredLogLevel, "s3_backup_log_level", "LogOff", "determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors.") fs.StringVar(&sse, "s3_backup_server_side_encryption", "", "server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file).") - fs.Int64Var(&minimumPartSize, "s3_backup_aws_minimum_partsize", 1024*1024*5, "Minimum part size to use") + fs.Int64Var(&minPartSize, "s3_backup_aws_min_partsize", manager.MinUploadPartSize, "Minimum part size to use, defaults to 5MiB but can be increased due to the dataset size.") } func init() { @@ -118,8 +123,6 @@ type logNameToLogLevel map[string]aws.ClientLogMode var logNameMap logNameToLogLevel -const sseCustomerPrefix = "sse_c:" - type endpointResolver struct { r s3.EndpointResolverV2 endpoint *string @@ -239,13 +242,13 @@ func calculateUploadPartSize(filesize int64) (partSizeBytes int64, err error) { } } - if minimumPartSize != 0 && partSizeBytes < minimumPartSize { - if minimumPartSize > 1024*1024*1024*5 || minimumPartSize < 1024*1024*5 { // 5GiB and 5MiB respectively + if minPartSize != 0 && partSizeBytes < minPartSize { + if minPartSize > 1024*1024*1024*5 || minPartSize < 1024*1024*5 { // 5GiB and 5MiB respectively return 0, fmt.Errorf("%w, currently set to %s", - ErrPartSize, humanize.IBytes(uint64(minimumPartSize)), + ErrPartSize, humanize.IBytes(uint64(minPartSize)), ) } - partSizeBytes = int64(minimumPartSize) + partSizeBytes = int64(minPartSize) } return diff --git a/go/vt/mysqlctl/s3backupstorage/s3_mock.go b/go/vt/mysqlctl/s3backupstorage/s3_mock.go index f244c4d63b1..910a22bd9d5 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3_mock.go +++ b/go/vt/mysqlctl/s3backupstorage/s3_mock.go @@ -162,7 +162,11 @@ func FailFirstWrite(s3bh *S3BackupHandle, ctx context.Context, filename string, return nil, fmt.Errorf("AddFile cannot be called on read-only backup") } - partSizeBytes := calculateUploadPartSize(filesize) + partSizeBytes, err := calculateUploadPartSize(filesize) + if err != nil { + return nil, err + } + reader, writer := io.Pipe() r := io.Reader(reader) @@ -181,7 +185,11 @@ func FailAllWrites(s3bh *S3BackupHandle, ctx context.Context, filename string, f return nil, fmt.Errorf("AddFile cannot be called on read-only backup") } - partSizeBytes := calculateUploadPartSize(filesize) + partSizeBytes, err := calculateUploadPartSize(filesize) + if err != nil { + return nil, err + } + reader, writer := io.Pipe() r := &failReadPipeReader{PipeReader: reader} diff --git a/go/vt/mysqlctl/s3backupstorage/s3_test.go b/go/vt/mysqlctl/s3backupstorage/s3_test.go index 033207bfdb0..5e9364219af 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3_test.go +++ b/go/vt/mysqlctl/s3backupstorage/s3_test.go @@ -329,9 +329,9 @@ func TestWithParams(t *testing.T) { assert.NotNil(t, s3.transport.Proxy) } -func TestGetPartSize(t *testing.T) { - originalMinimum := minimumPartSize - defer func() { minimumPartSize = originalMinimum }() +func TestCalculateUploadPartSize(t *testing.T) { + originalMinimum := minPartSize + defer func() { minPartSize = originalMinimum }() tests := []struct { name string @@ -386,8 +386,8 @@ func TestGetPartSize(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - minimumPartSize = tt.minimumPartSize - partSize, err := getPartSize(tt.filesize) + minPartSize = tt.minimumPartSize + partSize, err := calculateUploadPartSize(tt.filesize) require.ErrorIs(t, err, tt.err) require.Equal(t, tt.want, partSize) }) From 6a7a622a2fd32b6c70979a76475915cf9412b63d Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 12 Dec 2024 06:27:04 -0800 Subject: [PATCH 5/8] go mod tidy Signed-off-by: Renan Rangel --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6fd800b80ab..0aae178a606 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3 github.com/aws/smithy-go v1.22.0 github.com/bndr/gotabulate v1.1.2 + github.com/dustin/go-humanize v1.0.1 github.com/gammazero/deque v0.2.1 github.com/google/safehtml v0.1.0 github.com/hashicorp/go-version v1.7.0 @@ -153,7 +154,6 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/envoyproxy/go-control-plane v0.13.1 // indirect From 4ab78e0cdb04d9a5ea50e1710c5c7d219956f5a3 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 13 Dec 2024 02:41:29 -0800 Subject: [PATCH 6/8] use constants Signed-off-by: Renan Rangel --- go/vt/mysqlctl/s3backupstorage/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index c759108583b..c3a48fb0c74 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -243,7 +243,7 @@ func calculateUploadPartSize(filesize int64) (partSizeBytes int64, err error) { } if minPartSize != 0 && partSizeBytes < minPartSize { - if minPartSize > 1024*1024*1024*5 || minPartSize < 1024*1024*5 { // 5GiB and 5MiB respectively + if minPartSize > MaxPartSize || minPartSize < manager.MinUploadPartSize { // 5GiB and 5MiB respectively return 0, fmt.Errorf("%w, currently set to %s", ErrPartSize, humanize.IBytes(uint64(minPartSize)), ) From 364f8f5d5f26a05c2e29797532d558e4779b754b Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Mon, 16 Dec 2024 02:59:24 -0800 Subject: [PATCH 7/8] update operator.yaml Signed-off-by: Renan Rangel --- examples/operator/operator.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/operator/operator.yaml b/examples/operator/operator.yaml index 4b1b64df1ac..ded85de5285 100644 --- a/examples/operator/operator.yaml +++ b/examples/operator/operator.yaml @@ -679,6 +679,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -1995,6 +1998,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -3510,6 +3516,14 @@ spec: mysql80Compatible: type: string type: object + mysqldExporter: + type: string + vtbackup: + type: string + vtorc: + type: string + vttablet: + type: string type: object name: maxLength: 63 @@ -5241,6 +5255,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -6688,6 +6705,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string From 0342e4c2f8c9d08534cc8172bc7881739a6428aa Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 18 Dec 2024 05:31:54 -0800 Subject: [PATCH 8/8] minor comment update Signed-off-by: Renan Rangel --- go/vt/mysqlctl/s3backupstorage/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index c3a48fb0c74..4dd583009aa 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -228,7 +228,7 @@ func (bh *S3BackupHandle) handleAddFile(ctx context.Context, filename string, pa }() } -// this is a helper to calculate the part size, taking into consideration the minimum part size +// calculateUploadPartSize is a helper to calculate the part size, taking into consideration the minimum part size // passed in by an operator. func calculateUploadPartSize(filesize int64) (partSizeBytes int64, err error) { // Calculate s3 upload part size using the source filesize