Local source code feature (using bundle images)

Reference shipwright-io/build#717

Add `bundle` package that contains convenience code to push a local
source code directory as a bundle image to a container registry.

Add new flags for container image and local directory settings.

Add `github.com/google/go-containerregistry`.

Add `github.com/schollz/progressbar/v3`.

go.mod

@@ -3,20 +3,23 @@ module github.com/shipwright-io/cli
 go 1.15
 
 require (
+	github.com/google/go-containerregistry v0.5.1
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.8.2
 	github.com/shipwright-io/build v0.4.0
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
-	k8s.io/api v0.19.7
-	k8s.io/apimachinery v0.19.7
-	k8s.io/cli-runtime v0.19.7
-	k8s.io/client-go v12.0.0+incompatible
+	k8s.io/api v0.21.2
+	k8s.io/apimachinery v0.21.2
+	k8s.io/cli-runtime v0.21.2
+	k8s.io/client-go v0.21.2
 	k8s.io/utils v0.0.0-20210629042839-4a2b36d8d73f
 )
 
-replace k8s.io/client-go => k8s.io/client-go v0.19.7
-
 replace github.com/go-openapi/spec => github.com/go-openapi/spec v0.19.3 // Needed, otherwise we will hit this https://github.com/knative/client/pull/1207#issuecomment-770845105
+
+// TODO Drop replace once API changes are in shipwright-io/build
+replace github.com/shipwright-io/build => ../build

pkg/shp/bundle/bundle.go

@@ -0,0 +1,79 @@
+package bundle
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	progressbar "github.com/schollz/progressbar/v3"
+	"github.com/shipwright-io/build/pkg/reconciler/buildrun/resources/sources"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+)
+
+func Push(ctx context.Context, io *genericclioptions.IOStreams, localDirectory string, targetImage string) error {
+	tag, err := name.NewTag(targetImage)
+	if err != nil {
+		return err
+	}
+
+	auth, err := authn.DefaultKeychain.Resolve(tag.Context())
+	if err != nil {
+		return err
+	}
+
+	updates := make(chan v1.Update, 1)
+	done := make(chan struct{}, 1)
+	go func() {
+		var progress *progressbar.ProgressBar
+		for {
+			select {
+			case <-ctx.Done():
+				return
+
+			case <-done:
+				return
+
+			case update, ok := <-updates:
+				if !ok {
+					return
+				}
+
+				if progress == nil {
+					progress = progressbar.NewOptions(int(update.Total),
+						progressbar.OptionSetWriter(io.ErrOut),
+						progressbar.OptionEnableColorCodes(true),
+						progressbar.OptionShowBytes(true),
+						progressbar.OptionSetWidth(15),
+						progressbar.OptionSetPredictTime(false),
+						progressbar.OptionSetDescription("Uploading local source..."),
+						progressbar.OptionSetTheme(progressbar.Theme{
+							Saucer:        "[green]=[reset]",
+							SaucerHead:    "[green]>[reset]",
+							SaucerPadding: " ",
+							BarStart:      "[",
+							BarEnd:        "]"}),
+						progressbar.OptionClearOnFinish(),
+					)
+					defer progress.Close()
+				}
+
+				progress.ChangeMax64(update.Total)
+				progress.Set64(update.Complete)
+			}
+		}
+	}()
+
+	fmt.Fprintf(io.Out, "Bundling %q as %q ...\n", localDirectory, targetImage)
+	_, err = sources.Bundle(
+		tag,
+		localDirectory,
+		remote.WithAuth(auth),
+		remote.WithProgress(updates),
+	)
+
+	done <- struct{}{}
+	return err
+}

pkg/shp/cmd/build/create.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1"
+	"github.com/shipwright-io/cli/pkg/shp/bundle"
 	"github.com/shipwright-io/cli/pkg/shp/cmd/runner"
 	"github.com/shipwright-io/cli/pkg/shp/flags"
 	"github.com/shipwright-io/cli/pkg/shp/params"
@@ -16,8 +17,9 @@ import (
 type CreateCommand struct {
 	cmd *cobra.Command // cobra command instance
 
-	name      string                   // build resource's name
-	buildSpec *buildv1alpha1.BuildSpec // stores command-line flags
+	name           string // build resource's name
+	localSourceDir *string
+	buildSpec      *buildv1alpha1.BuildSpec // stores command-line flags
 }
 
 const buildCreateLongDesc = `
@@ -47,6 +49,19 @@ func (c *CreateCommand) Validate() error {
 	if c.name == "" {
 		return fmt.Errorf("name must be provided")
 	}
+
+	if c.buildSpec.Source.URL != "" &&
+		c.buildSpec.Source.Container != nil &&
+		c.buildSpec.Source.Container.Image != "" {
+		return fmt.Errorf("both source URL and container image are specified, only one can be used at the same time")
+	}
+
+	if c.buildSpec.Source.URL == "" &&
+		c.buildSpec.Source.Container != nil &&
+		c.buildSpec.Source.Container.Image == "" {
+		return fmt.Errorf("no input source was specified, either source URL or container image needs to be provided")
+	}
+
 	return nil
 }
 
@@ -55,6 +70,15 @@ func (c *CreateCommand) Run(params *params.Params, io *genericclioptions.IOStrea
 	b := &buildv1alpha1.Build{Spec: *c.buildSpec}
 	flags.SanitizeBuildSpec(&b.Spec)
 
+	// Local source code mode:
+	// Make sure to bundle the configured local source directory into an image
+	// bundle and push it to the provided target registry.
+	if b.Spec.Source.Container.Image != "" {
+		if err := bundle.Push(c.cmd.Context(), io, *c.localSourceDir, b.Spec.Source.Container.Image); err != nil {
+			return err
+		}
+	}
+
 	buildResource := resource.GetBuildResource(params)
 	if err := buildResource.Create(c.cmd.Context(), c.name, b); err != nil {
 		return err
@@ -74,15 +98,13 @@ func createCmd() runner.SubCommand {
 	// instantiating command-line flags and the build-spec structure which receives the informed flag
 	// values, also marking certain flags as mandatory
 	buildSpecFlags := flags.BuildSpecFromFlags(cmd.Flags())
-	if err := cmd.MarkFlagRequired(flags.SourceURLFlag); err != nil {
-		panic(err)
-	}
 	if err := cmd.MarkFlagRequired(flags.OutputImageFlag); err != nil {
 		panic(err)
 	}
 
 	return &CreateCommand{
-		cmd:       cmd,
-		buildSpec: buildSpecFlags,
+		cmd:            cmd,
+		localSourceDir: flags.BundleLocalSourceFlag(cmd.Flags()),
+		buildSpec:      buildSpecFlags,
 	}
 }

pkg/shp/cmd/buildrun/create.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1"
+	"github.com/shipwright-io/cli/pkg/shp/bundle"
 	"github.com/shipwright-io/cli/pkg/shp/cmd/runner"
 	"github.com/shipwright-io/cli/pkg/shp/flags"
 	"github.com/shipwright-io/cli/pkg/shp/params"
@@ -16,8 +17,9 @@ import (
 type CreateCommand struct {
 	cmd *cobra.Command // cobra command instance
 
-	name         string                      // buildrun name
-	buildRunSpec *buildv1alpha1.BuildRunSpec // stores command-line flags
+	name           string // buildrun name
+	localSourceDir *string
+	buildRunSpec   *buildv1alpha1.BuildRunSpec // stores command-line flags
 }
 
 const buildRunCreateLongDesc = `
@@ -56,6 +58,19 @@ func (c *CreateCommand) Run(params *params.Params, ioStreams *genericclioptions.
 	br := &buildv1alpha1.BuildRun{Spec: *c.buildRunSpec}
 	flags.SanitizeBuildRunSpec(&br.Spec)
 
+	// Local source code mode:
+	// Make sure to bundle the configured local source directory into an image
+	// bundle and push it to the provided target registry.
+	var build = buildv1alpha1.Build{}
+	if err := resource.GetBuildResource(params).Get(c.cmd.Context(), c.buildRunSpec.BuildRef.Name, &build); err != nil {
+		return fmt.Errorf("failed to get referenced build %q: %w", c.buildRunSpec.BuildRef.Name, err)
+	}
+	if build.Spec.Source.Container.Image != "" {
+		if err := bundle.Push(c.cmd.Context(), ioStreams, *c.localSourceDir, build.Spec.Source.Container.Image); err != nil {
+			return err
+		}
+	}
+
 	buildRunResource := resource.GetBuildRunResource(params)
 	if err := buildRunResource.Create(c.cmd.Context(), c.name, br); err != nil {
 		return err
@@ -81,7 +96,8 @@ func createCmd() runner.SubCommand {
 	}
 
 	return &CreateCommand{
-		cmd:          cmd,
-		buildRunSpec: buildRunSpecFlags,
+		cmd:            cmd,
+		localSourceDir: flags.BundleLocalSourceFlag(cmd.Flags()),
+		buildRunSpec:   buildRunSpecFlags,
 	}
 }

pkg/shp/cmd/buildrun/list.go

@@ -7,6 +7,7 @@ import (
 
 	buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1"
 	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 
@@ -72,8 +73,13 @@ func (c *ListCommand) Run(params *params.Params, io *genericclioptions.IOStreams
 
 	for _, br := range brs.Items {
 		name := br.Name
-		status := br.Status.Reason
-
+		status := string(metav1.ConditionUnknown)
+		for _, condition := range br.Status.Conditions {
+			if condition.Type == buildv1alpha1.Succeeded {
+				status = condition.Reason
+				break
+			}
+		}
 		fmt.Fprintf(writer, columnTemplate, name, status)
 	}
 

pkg/shp/flags/build.go

@@ -1,6 +1,8 @@
 package flags
 
 import (
+	"os"
+
 	buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1"
 	"github.com/spf13/pflag"
 	corev1 "k8s.io/api/core/v1"
@@ -16,6 +18,7 @@ func BuildSpecFromFlags(flags *pflag.FlagSet) *buildv1alpha1.BuildSpec {
 			Credentials: &corev1.LocalObjectReference{},
 			Revision:    pointer.String(""),
 			ContextDir:  pointer.String(""),
+			Container:   &buildv1alpha1.Container{},
 		},
 		Strategy: &buildv1alpha1.Strategy{
 			Kind:       &clusterBuildStrategyKind,
@@ -49,6 +52,11 @@ func SanitizeBuildSpec(b *buildv1alpha1.BuildSpec) {
 	if b.Source.Credentials != nil && b.Source.Credentials.Name == "" {
 		b.Source.Credentials = nil
 	}
+
+	if b.Source.Container != nil && b.Source.Container.Image == "" {
+		b.Source.Container = nil
+	}
+
 	if b.Builder != nil {
 		if b.Builder.Credentials != nil && b.Builder.Credentials.Name == "" {
 			b.Builder.Credentials = nil
@@ -58,3 +66,16 @@ func SanitizeBuildSpec(b *buildv1alpha1.BuildSpec) {
 		}
 	}
 }
+
+func BundleLocalSourceFlag(flags *pflag.FlagSet) (result *string) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		cwd = "."
+	}
+
+	return flags.String(
+		"source-directory",
+		cwd,
+		"directory to be used for local source code",
+	)
+}

pkg/shp/flags/build_test.go

@@ -17,14 +17,14 @@ import (
 func TestBuildSpecFromFlags(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 
-	credentials := corev1.LocalObjectReference{Name: "name"}
 	buildStrategyKind := buildv1alpha1.ClusterBuildStrategyKind
 	expected := &buildv1alpha1.BuildSpec{
 		Source: buildv1alpha1.Source{
-			Credentials: &credentials,
+			Credentials: &corev1.LocalObjectReference{Name: "source-credentials"},
 			URL:         "https://some.url",
 			Revision:    pointer.String("some-rev"),
 			ContextDir:  pointer.String("some-contextdir"),
+			Container:   &buildv1alpha1.Container{},
 		},
 		Strategy: &buildv1alpha1.Strategy{
 			Name:       "strategy-name",
@@ -33,11 +33,11 @@ func TestBuildSpecFromFlags(t *testing.T) {
 		},
 		Dockerfile: pointer.String("some-dockerfile"),
 		Builder: &buildv1alpha1.Image{
-			Credentials: &credentials,
+			Credentials: &corev1.LocalObjectReference{Name: "builder-credentials"},
 			Image:       "builder-image",
 		},
 		Output: buildv1alpha1.Image{
-			Credentials: &credentials,
+			Credentials: &corev1.LocalObjectReference{Name: "output-credentials"},
 			Image:       "output-image",
 		},
 		Timeout: &metav1.Duration{

pkg/shp/flags/flags.go

@@ -22,6 +22,8 @@ const (
 	SourceURLFlag = "source-url"
 	// SourceRevisionFlag command-line flag.
 	SourceRevisionFlag = "source-revision"
+	// SourceContainerImageFlag command-line flag.
+	SourceContainerImageFlag = "source-bundle-image"
 	// SourceContextDirFlag command-line flag.
 	SourceContextDirFlag = "source-context-dir"
 	// SourceCredentialsSecretFlag command-line flag.
@@ -58,6 +60,12 @@ func sourceFlags(flags *pflag.FlagSet, source *buildv1alpha1.Source) {
 		"",
 		"git repository source revision",
 	)
+	flags.StringVar(
+		&source.Container.Image,
+		SourceContainerImageFlag,
+		"",
+		"source code bundle image",
+	)
 	flags.StringVar(
 		source.ContextDir,
 		SourceContextDirFlag,

pkg/shp/resource/resource_test.go

@@ -46,6 +46,8 @@ func TestResource(t *testing.T) {
 	})
 
 	t.Run("Resource List", func(t *testing.T) {
+		// See https://github.com/kubernetes/client-go/issues/983
+		t.Skip("k8s 1.20: resource.List fails, can't assign or convert unstructured.Unstructured into v1alpha1.Build")
 		var buildList buildv1alpha1.BuildList
 		err = buildResource.List(context.TODO(), &buildList)