Image should support pulling artifacts from docker images to build the context

api/v1alpha1/images.go

@@ -1,6 +1,11 @@
 package v1alpha1
 
-import "k8s.io/apimachinery/pkg/runtime/schema"
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
 
 var (
 	ImageGVK = schema.FromAPIVersionAndKind(Group+"/"+Version, "Image")
@@ -29,12 +34,27 @@ type ImageSpec struct {
 	// Image is the full path of the image to be built
 	// e.g.us-west1-docker.pkg.dev/dev-sailplane/images/hydros/agent
 	// So it includes the registry and repository but not the tag or digest
-	Image   string           `yaml:"image,omitempty"`
-	Source  []*Source        `yaml:"source,omitempty"`
+	Image string `yaml:"image,omitempty"`
+	// Source are the source for the image
+	Source  []*ImageSource   `yaml:"source,omitempty"`
 	Builder *ArtifactBuilder `yaml:"builder,omitempty"`
 }
 
-// Source is a local path to include as an artifact.
+type ImageSource struct {
+	// URI is the path of the resource to use as a source
+	// This can be a local path or a docker image. If its a local path relative paths will be interpreted
+	// relative to the location of the YAML file containing the resource.
+	// e.g.us-west1-docker.pkg.dev/dev-sailplane/images/hydros/agent
+	//
+	// Use file:// to specify a local path e.g. file:///path/to/dir. Note the third "/" indicates its an absolute path
+	// If its "//" then its a relative path. I'm not sure it makes sense to support relative paths because what
+	// would they be relative to?
+	// TODO(jeremy): If the tag isn't specified we should look for the same tag at which the new image is being built
+	URI      string           `yaml:"uri,omitempty"`
+	Mappings []*SourceMapping `yaml:"mappings,omitempty"`
+}
+
+// SourceMapping specifies how source files are mapped into the destination artifact
 // It is inspired by skaffold; https://skaffold.dev/docs/references/yaml/
 // When building images from a YAML file the src is a relative path to the location of the YAML file.
 // src can start with a parent prefix e.g. ".." to obtain files higher in the directory tree relative to the
@@ -44,7 +64,7 @@ type ImageSpec struct {
 // /p/b/file.txt
 // And image.yaml contains src "../b/file.txt" then the file will be copied to b/file.txt by default in the
 // tarball
-type Source struct {
+type SourceMapping struct {
 	// Src is a glob pattern to match local paths against. Directories should be delimited by / on all platforms.
 	// e.g. "css/**/*.css"
 	Src string `yaml:"src,omitempty"`
@@ -81,6 +101,10 @@ type GCBConfig struct {
 	// See: https://cloud.google.com/build/docs/api/reference/rest/v1/projects.builds#machinetype
 	// For values. UNSPECIFIED uses the default value which has 1 CPU
 	MachineType string `yaml:"machineType,omitempty"`
+
+	// Dockerfile is the path to the Dockerfile to use for building the image
+	// This should be the path inside the context
+	Dockerfile string `yaml:"dockerfile,omitempty"`
 }
 
 type ImageStatus struct {
@@ -91,3 +115,35 @@ type ImageStatus struct {
 	// SHA is the SHA of the image
 	SHA string `yaml:"sha,omitempty"`
 }
+
+// IsValid returns true if the config is valid.
+// For invalid config the string will be a message of validation errors
+func (c *Image) IsValid() (string, bool) {
+	errors := make([]string, 0, 10)
+
+	if c.Spec.Image == "" {
+		errors = append(errors, "Image must be specified")
+	}
+
+	for i, source := range c.Spec.Source {
+		if source.URI == "" {
+			errors = append(errors, fmt.Sprintf("Source[%d].URI must be specified", i))
+		}
+		if len(source.Mappings) == 0 {
+			errors = append(errors, fmt.Sprintf("Source[%d].Mappings must be specified", i))
+		}
+	}
+
+	if c.Spec.Builder.GCB.Bucket == "" {
+		errors = append(errors, "Spec.Builder.GCB.Bucket must be specified")
+	}
+
+	if c.Spec.Builder.GCB.Project == "" {
+		errors = append(errors, "Spec.Builder.GCB.Project must be specified")
+	}
+
+	if len(errors) > 0 {
+		return "Image is invalid. " + strings.Join(errors, ". "), false
+	}
+	return "", true
+}

cmd/commands/apply.go

@@ -32,7 +32,7 @@ type applyOptions struct {
 func NewApplyCmd() *cobra.Command {
 	aOptions := applyOptions{}
 
-	// TODO(jeremy): We should update apply to support the Image resource.
+	// TODO(jeremy): We should update apply to support the image resource.
 	applyCmd := &cobra.Command{
 		Use:   "apply <resource.yaml> <resourceDir> <resource.yaml> ...",
 		Short: "Apply the specified resource.",

docs/image_build.md

@@ -14,27 +14,21 @@ Here's an example
 kind: Image
 apiVersion: hydros.sailplane.ai/v1alpha1
 metadata:
-  name: kp
-  namespace: kubepilot
+  name: hydros
+  namespace: hydros
 spec:
   image: us-west1-docker.pkg.dev/dev-sailplane/images/hydros/hydros
   source:
-    # Leave it at the root of the context because that's what hydros will look for.
-    - src: Dockerfile
-    # Specify individual directories so we don't include hidden directories
-    - src: "go.mod"
-      dest: "kubedr"
-    - src: "go.sum"
-      dest: "kubedr"
-    - src: "api/**/*.go"
-      dest: "kubedr"
-    - src: "cmd/**/*.go"
-      dest: "kubedr"
-    - src: "pkg/**/*"
-      dest: "kubedr"
-    - src: "test/**/*.go"
-      dest: "kubedr"
-    - src: "../vendor/**/*"
+    - uri: https://github.com/jlewi/hydros.git
+      mappings:
+        - src: Dockerfile
+        # Specify individual directories so we don't include hidden directories
+        - src: "go.mod"
+        - src: "go.sum"
+        - src: "api/**/*.go"
+        - src: "cmd/**/*.go"
+        - src: "pkg/**/*.go"
+        - src: "test/**/*.go"
   builder:
     gcb:
       project: dev-sailplane
@@ -48,27 +42,26 @@ Currently only the GCB builder is supported.
 The context for the image is defined by the source field. Each entry in the source field specifies files
 that will be copied into the context. The source field is an array of objects with the following fields:
 
+* uri: The URI of the source resource. This can be a git repository or docker image.
+  * For git repositories the URI should be the URL of the repository e.g. `https://github.com/jlewi/hydros.git`
+  * For docker images the URI should be the image name with the scheme `docker://` e.g. `docker://gcr.io/dev-sailplane/hydros:latest
+* mappings: An array of mappings specifying files to be copied into the context.
+
 * src: This is a glob expression matching files to be copied into the context. The glob expression is relative to the
-  directory containing the image definition file.
+  root of the resource (e.g. the repository or the docker image). The following glob expressions are supported:
   * Double star `**` can be used to match all subdirectories
   * You can use `..` to go up the directory tree to match files located in parent directories of the `.yaml` file
 * dest: This is the destination directory for the files. 
 * strip: This is a prefix to strip of the matched files when computing the location in the destination directory. 
 
 The location of the files inside the produced context (tarball) is as follows
 
-```
-basePath = dir(imageDefinitionFile)
-rPath = path of matched file relative to basePath
-strippedRPath = strip rPath of prefix 
-destPath = dest + stripPipedRpath
-```
-
-In the case where `src` begins with `..` basePath is adjusted to be the parent directory.
+Typically the first source will be the git repository containing the source code.
 
 ### Dockerfile
 
-Hydros currently requires the Dockerfile to be named `Dockerfile` and located at the root of the context.
+By default Hydros assumes the Dockerfile to be named `Dockerfile` and located at the root of the context. However,
+you can specify the path to the Dockerfile using the `dockerfile` field in the `gcb` section.
 
 ### Docker build args
 
@@ -93,4 +86,7 @@ To build an image you can use the `hydros build` command
 hydros build ~/git_roboweb/kubedr/image.yaml
 ```
 
-If an image already exists in the registry with the same tag as the current commit, the image will not be rebuilt.
+* If an image already exists in the registry with the same tag as the current commit, the image will not be rebuilt.
+* If the repository is dirty Hydros will commit the changes and then build the image
+* Hydros will automatically detect if the file is located in a git repository that matches one of the sources and 
+  use the commit hash as the tag.

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.3
 	github.com/google/go-cmp v0.6.0
-	github.com/google/go-containerregistry v0.14.0
+	github.com/google/go-containerregistry v0.18.0
 	github.com/google/uuid v1.4.0
 	github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2
 	github.com/otiai10/copy v1.6.0
@@ -81,7 +81,7 @@ require (
 	cloud.google.com/go/iam v1.1.3 // indirect
 	github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
 	github.com/DataDog/sketches-go v1.0.0 // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -91,9 +91,9 @@ require (
 	github.com/cloudflare/circl v1.1.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/cli v23.0.1+incompatible // indirect
-	github.com/docker/distribution v2.8.2-beta.1+incompatible // indirect
-	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/cli v24.0.0+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/docker/docker v24.0.0+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
@@ -127,7 +127,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
@@ -144,7 +144,7 @@ require (
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.12.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -154,7 +154,7 @@ require (
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sirupsen/logrus v1.9.1 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
@@ -163,7 +163,7 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e // indirect
 	github.com/tinylib/msgp v1.1.2 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
+	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
@@ -178,7 +178,7 @@ require (
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.150.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect