Transfer - Send chunks if they are bigger than GiB (#966)

artifactory/commands/transferfiles/api/types.go

@@ -23,6 +23,10 @@ const (
 	Phase1 int = 0
 	Phase2 int = 1
 	Phase3 int = 2
+
+	maxFilesInChunk = 16
+	// 1 GiB
+	maxBytesInChunk = 1 << 30
 )
 
 type TargetAuth struct {
@@ -101,3 +105,18 @@ func (uc *UploadChunk) AppendUploadCandidateIfNeeded(file FileRepresentation, bu
 	}
 	uc.UploadCandidates = append(uc.UploadCandidates, file)
 }
+
+// Return true if the chunk contains at least 16 files or at least 1GiB in total
+func (uc *UploadChunk) IsChunkFull() bool {
+	if len(uc.UploadCandidates) >= maxFilesInChunk {
+		return true
+	}
+	var totalSize int64 = 0
+	for _, uploadCandidate := range uc.UploadCandidates {
+		totalSize += uploadCandidate.Size
+		if totalSize > maxBytesInChunk {
+			return true
+		}
+	}
+	return false
+}

artifactory/commands/transferfiles/api/types_test.go

@@ -0,0 +1,51 @@
+package api
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAppendUploadCandidateIfNeeded(t *testing.T) {
+	uploadChunk := &UploadChunk{}
+
+	// Regular file
+	uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "regular-file"}, false)
+	assert.Len(t, uploadChunk.UploadCandidates, 1)
+
+	// Build info
+	uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "build-info.json"}, true)
+	assert.Len(t, uploadChunk.UploadCandidates, 2)
+
+	// Directory in build info - should be skipped
+	uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{}, true)
+	assert.Len(t, uploadChunk.UploadCandidates, 2)
+}
+
+var isChunkFullCases = []struct {
+	files  []FileRepresentation
+	isFull bool
+}{
+	{[]FileRepresentation{}, false},
+	{[]FileRepresentation{{Name: "slim-jim", Size: 10737418}}, false},
+	{[]FileRepresentation{{Name: "fat-vinny", Size: 1073741825}}, true},
+}
+
+func TestIsChunkFull(t *testing.T) {
+	for _, testCase := range isChunkFullCases {
+		t.Run("", func(t *testing.T) {
+			uploadChunk := &UploadChunk{UploadCandidates: testCase.files}
+			assert.Equal(t, testCase.isFull, uploadChunk.IsChunkFull())
+		})
+	}
+}
+
+func TestIsChunkFullNumberOfFiles(t *testing.T) {
+	uploadChunk := &UploadChunk{}
+	for i := 0; i < maxFilesInChunk; i++ {
+		assert.False(t, uploadChunk.IsChunkFull())
+		uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: fmt.Sprintf("%d", i)}, false)
+	}
+	assert.True(t, uploadChunk.IsChunkFull())
+}

artifactory/commands/transferfiles/fulltransfer.go

@@ -239,7 +239,7 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper producerConsumerWrapper,
 		return
 	}
 	curUploadChunk.AppendUploadCandidateIfNeeded(file, m.buildInfoRepo)
-	if len(curUploadChunk.UploadCandidates) == uploadChunkSize {
+	if curUploadChunk.IsChunkFull() {
 		_, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&m.phaseBase, *curUploadChunk, uploadChunkChan, errorsChannelMng), pcWrapper.errorsQueue.AddError)
 		if err != nil {
 			return

artifactory/commands/transferfiles/transfer.go

@@ -28,7 +28,6 @@ import (
 )
 
 const (
-	uploadChunkSize = 16
 	// Size of the channel where the transfer's go routines write the transfer errors
 	fileWritersChannelSize       = 500000
 	retries                      = 600

artifactory/commands/transferfiles/utils.go

@@ -419,7 +419,7 @@ func uploadByChunks(files []api.FileRepresentation, uploadTokensChan chan Upload
 			continue
 		}
 		curUploadChunk.AppendUploadCandidateIfNeeded(file, base.buildInfoRepo)
-		if len(curUploadChunk.UploadCandidates) == uploadChunkSize {
+		if curUploadChunk.IsChunkFull() {
 			_, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&base, curUploadChunk, uploadTokensChan, errorsChannelMng), pcWrapper.errorsQueue.AddError)
 			if err != nil {
 				return