feat: add support for delta package uploads using go-octodiff (#226)

go.mod

@@ -5,7 +5,8 @@ go 1.21
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.6
 	github.com/MakeNowJust/heredoc/v2 v2.0.1
-	github.com/OctopusDeploy/go-octopusdeploy/v2 v2.38.1
+	github.com/OctopusDeploy/go-octodiff v1.0.0
+	github.com/OctopusDeploy/go-octopusdeploy/v2 v2.45.0
 	github.com/bmatcuk/doublestar/v4 v4.4.0
 	github.com/briandowns/spinner v1.19.0
 	github.com/google/uuid v1.3.0
@@ -18,7 +19,7 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.14.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2
 	golang.org/x/term v0.4.0
 )

go.sum

@@ -44,8 +44,12 @@ github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZ
 github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
-github.com/OctopusDeploy/go-octopusdeploy/v2 v2.38.1 h1:XxaJaWRD+dyTYKkY+Rbw1kZS9G3KkNECvn2Hmk5Bsls=
-github.com/OctopusDeploy/go-octopusdeploy/v2 v2.38.1/go.mod h1:GZmFu6LmN8Yg0tEoZx3ytk9FnaH+84cWm7u5TdWZC6E=
+github.com/OctopusDeploy/go-octodiff v1.0.0 h1:U+ORg6azniwwYo+O44giOw6TiD5USk8S4VDhOQ0Ven0=
+github.com/OctopusDeploy/go-octodiff v1.0.0/go.mod h1:Mze0+EkOWTgTmi8++fyUc6r0aLZT7qD9gX+31t8MmIU=
+github.com/OctopusDeploy/go-octopusdeploy/v2 v2.44.2-0.20240629192925-20edc738f146 h1:YgOokmzFxYxwW3Gab7Y4zgDJtSjPDPzBJZPb92ZueGI=
+github.com/OctopusDeploy/go-octopusdeploy/v2 v2.44.2-0.20240629192925-20edc738f146/go.mod h1:ggvOXzMnq+w0pLg6C9zdjz6YBaHfO3B3tqmmB7JQdaw=
+github.com/OctopusDeploy/go-octopusdeploy/v2 v2.45.0 h1:eTNAHSimCL5d0vTawIEB7TZJQ6Pt8KhcB8wUHS6rwq8=
+github.com/OctopusDeploy/go-octopusdeploy/v2 v2.45.0/go.mod h1:ggvOXzMnq+w0pLg6C9zdjz6YBaHfO3B3tqmmB7JQdaw=
 github.com/bmatcuk/doublestar/v4 v4.4.0 h1:LmAwNwhjEbYtyVLzjcP/XeVw4nhuScHGkF/XWXnvIic=
 github.com/bmatcuk/doublestar/v4 v4.4.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/briandowns/spinner v1.19.0 h1:s8aq38H+Qju89yhp89b4iIiMzMm8YN3p6vGpwyh/a8E=
@@ -249,8 +253,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
 github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/cmd/package/upload/upload.go

@@ -4,13 +4,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/OctopusDeploy/cli/pkg/apiclient"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/OctopusDeploy/cli/pkg/apiclient"
-
 	"github.com/MakeNowJust/heredoc/v2"
 	"github.com/OctopusDeploy/cli/pkg/constants"
 	"github.com/OctopusDeploy/cli/pkg/constants/annotations"
@@ -30,8 +29,10 @@ const (
 	FlagAliasOverwrite     = "overwrite"
 	FlagAliasOverwriteMode = "overwritemode" // I keep forgetting the hyphen
 
-	// replace-existing deprected in the .NET CLI so not brought across
-	FlagUseDeltaCompression = "use-delta-compression" // this is not yet supported, but will be in future when we implement OctoDiff in go
+	FlagUseDeltaCompression = "use-delta-compression"
+	FlagAliasDelta          = "delta" // for less typing
+
+	// "replace-existing" flag deprecated in the .NET CLI so not brought across
 
 	FlagContinueOnError = "continue-on-error"
 )
@@ -57,14 +58,16 @@ func NewCmdUpload(f factory.Factory) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "upload",
 		Short:   "upload one or more packages to Octopus Deploy",
-		Long:    "upload one or more packages to Octopus Deploy. Glob patterns are supported.",
+		Long:    "upload one or more packages to Octopus Deploy. Glob patterns are supported. Delta compression is off by default.",
 		Aliases: []string{"push"},
 		Example: heredoc.Docf(`
 			$ %[1]s package upload --package SomePackage.1.0.0.zip
 			$ %[1]s package upload SomePackage.1.0.0.tar.gz --overwrite-mode overwrite
 			$ %[1]s package push SomePackage.1.0.0.zip	
 			$ %[1]s package upload bin/**/*.zip --continue-on-error
 			$ %[1]s package upload PkgA.1.0.0.zip PkgB.2.0.0.tar.gz PkgC.1.0.0.nupkg
+			$ %[1]s package upload --package SomePackage.2.0.0.zip --use-delta-compression
+			$ %[1]s package upload SomePackage.2.0.0.zip --delta # alias for --use-delta-compression
 		`, constants.ExecutableName),
 		Annotations: map[string]string{annotations.IsCore: "true"},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -79,11 +82,16 @@ func NewCmdUpload(f factory.Factory) *cobra.Command {
 	flags := cmd.Flags()
 	flags.StringArrayVarP(&uploadFlags.Package.Value, uploadFlags.Package.Name, "p", nil, "Package to upload, may be specified multiple times. Any arguments without flags will be treated as packages")
 	flags.StringVarP(&uploadFlags.OverwriteMode.Value, uploadFlags.OverwriteMode.Name, "", "", "Action when a package already exists. Valid values are 'fail', 'overwrite', 'ignore'. Default is 'fail'")
-	flags.BoolVarP(&uploadFlags.ContinueOnError.Value, uploadFlags.ContinueOnError.Name, "", false, "When uploading multiple packages, controls whether the CLI continues after a failed upload. Default is to abort.")
+	flags.BoolVarP(&uploadFlags.ContinueOnError.Value, uploadFlags.ContinueOnError.Name, "", false, "When uploading multiple packages, controls whether the CLI continues after a failed upload. Default is to abort")
+	// note to future developers: Added in July 2024; We can remove the beta disclaimer on delta compression at the end of 2024 if customers haven't reported any issues.
+	// when removing the beta disclaimer, please leave delta compression off by default. It can help on slow networks, but can be slower over fast ones due to the additional work of diffing/patching.
+	// it's also useless for tar.gz's or zips with single files in them (e.g. someone puts a .MSI file in a .zip just so octopus will accept it)
+	flags.BoolVarP(&uploadFlags.UseDeltaCompression.Value, uploadFlags.UseDeltaCompression.Name, "d", false, "Attempt to use delta compression when uploading. Off by default. As a recent addition to the CLI, it should be considered in beta.")
 	flags.SortFlags = false
 
 	flagAliases := make(map[string][]string, 1)
 	util.AddFlagAliasesString(flags, FlagOverwriteMode, flagAliases, FlagAliasOverwrite, FlagAliasOverwriteMode)
+	util.AddFlagAliasesBool(flags, FlagUseDeltaCompression, flagAliases, FlagAliasDelta)
 
 	cmd.PreRunE = func(cmd *cobra.Command, args []string) error {
 		util.ApplyFlagAliases(cmd.Flags(), flagAliases)
@@ -136,14 +144,17 @@ func uploadRun(cmd *cobra.Command, f factory.Factory, flags *UploadFlags) error
 		return fmt.Errorf("invalid value '%s' for --overwrite-mode. Valid values are 'fail', 'ignore', 'overwrite'", overwriteMode)
 	}
 
+	useDeltaCompression := flags.UseDeltaCompression.Value
+
 	var jsonResult uploadViewModel
 	didErrorsOccur := false
 
 	// with globs it's easy to specify the same thing twice by accident, so keep track of what's been uploaded as we go
 	seenPackages := make(map[string]bool)
 	doUpload := func(path string) error {
 		if !seenPackages[path] {
-			created, err := uploadFileAtPath(octopus, space, path, resolvedOverwriteMode, cmd)
+			uploadResult, err := uploadFileAtPath(octopus, space, path, resolvedOverwriteMode, useDeltaCompression, cmd)
+
 			seenPackages[path] = true // whether a given package succeeds or fails, we still don't want to process it twice
 
 			if err != nil {
@@ -176,11 +187,32 @@ func uploadRun(cmd *cobra.Command, f factory.Factory, flags *UploadFlags) error
 				case constants.OutputFormatBasic:
 					cmd.Printf("%s\n", path)
 				default:
-					if created {
+					if uploadResult.CreatedNewFile {
 						cmd.Printf("Uploaded package %s\n", path)
 					} else {
 						cmd.Printf("Ignored existing package %s\n", path)
 					}
+
+					if uploadResult.UploadMethod == packages.UploadMethodDelta && uploadResult.UploadInfo != nil {
+						deltaInfo := uploadResult.UploadInfo
+						deltaRatio := 0.0
+						if deltaInfo.FileSize > 0 {
+							deltaRatio = float64(deltaInfo.DeltaSize) / float64(deltaInfo.FileSize) * 100
+						}
+
+						switch deltaInfo.DeltaBehaviour {
+						case packages.DeltaBehaviourNoPreviousFile:
+							cmd.Printf("    Full upload for package %s. No previous versions available\n", path)
+						case packages.DeltaBehaviourNotEfficient:
+							cmd.Printf("    Full upload for package %s. Delta size was %.1f%% of full file (too large)\n", path, deltaRatio)
+						case packages.DeltaBehaviourUploadedDeltaFile:
+							cmd.Printf("    Delta upload for package %s.\n"+
+								"    Delta size was %.1f%% of full file, saving %s\n",
+								path, deltaRatio, util.HumanReadableBytes(deltaInfo.FileSize-deltaInfo.DeltaSize))
+						default:
+							break // a future unknown DeltaBehaviour will result in printing nothing, deliberately
+						}
+					} // else delta disabled; print nothing
 				}
 			}
 		}
@@ -220,22 +252,22 @@ func uploadRun(cmd *cobra.Command, f factory.Factory, flags *UploadFlags) error
 	return nil
 }
 
-func uploadFileAtPath(octopus newclient.Client, space *spaces.Space, path string, overwriteMode packages.OverwriteMode, cmd *cobra.Command) (bool, error) {
-	opener := func(name string) (io.ReadCloser, error) { return os.Open(name) }
+func uploadFileAtPath(octopus newclient.Client, space *spaces.Space, path string, overwriteMode packages.OverwriteMode, useDeltaCompression bool, cmd *cobra.Command) (*packages.PackageUploadResponseV2, error) {
+	opener := func(name string) (io.ReadSeekCloser, error) { return os.Open(name) }
 	if cmd.Context() != nil { // allow context to override the definition of 'os.Open' for testing
-		if f, ok := cmd.Context().Value(constants.ContextKeyOsOpen).(func(string) (io.ReadCloser, error)); ok {
+		if f, ok := cmd.Context().Value(constants.ContextKeyOsOpen).(func(string) (io.ReadSeekCloser, error)); ok {
 			opener = f
 		}
 	}
 
 	fileReader, err := opener(path)
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
 	// Note: the PackageUploadResponse has a lot of information in it, but we've chosen not to do anything
 	// with it in the CLI at this time.
-	_, created, err := packages.Upload(octopus, space.ID, filepath.Base(path), fileReader, overwriteMode)
+	result, err := packages.UploadV2(octopus, space.ID, filepath.Base(path), fileReader, overwriteMode, useDeltaCompression)
 	_ = fileReader.Close()
-	return created, err
+	return result, err
 }