From 6d9270106b4e3649b1323bd240f28eef54dcb7cb Mon Sep 17 00:00:00 2001 From: Matt Nikkel Date: Thu, 13 Jan 2022 13:43:22 -0500 Subject: [PATCH] Add ImageTxt collection + storing logic --- go.mod | 2 +- go.sum | 4 +- pkg/collection/imagetxt/imagetxt.go | 232 ++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 pkg/collection/imagetxt/imagetxt.go diff --git a/go.mod b/go.mod index c9d1ea56..3b68343e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mholt/archiver/v3 v3.5.1 github.com/opencontainers/image-spec v1.0.2 github.com/pkg/errors v0.9.1 - github.com/rancherfederal/ocil v0.1.7 + github.com/rancherfederal/ocil v0.1.8 github.com/rs/zerolog v1.26.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 diff --git a/go.sum b/go.sum index 2e9bc4f8..8cc06f6d 100644 --- a/go.sum +++ b/go.sum @@ -869,8 +869,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rancherfederal/ocil v0.1.7 h1:qIEaMEoLPbNSKZpmllZnIAj7YuD1xq1f0tVFL+32IIs= -github.com/rancherfederal/ocil v0.1.7/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY= +github.com/rancherfederal/ocil v0.1.8 h1:jVYD/AY7ipXgKepdZDDG1mxMOxCk/KIDdZw2qsseR+c= +github.com/rancherfederal/ocil v0.1.8/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/pkg/collection/imagetxt/imagetxt.go b/pkg/collection/imagetxt/imagetxt.go new file mode 100644 index 00000000..8f573a13 --- /dev/null +++ b/pkg/collection/imagetxt/imagetxt.go @@ -0,0 +1,232 @@ +package imagetxt + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/rancherfederal/hauler/pkg/log" + + "github.com/google/go-containerregistry/pkg/name" + artifact "github.com/rancherfederal/ocil/pkg/artifacts" + "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" + "github.com/rancherfederal/ocil/pkg/artifacts/image" +) + +type ImageTxt struct { + Ref string + IncludeSources map[string]bool + ExcludeSources map[string]bool + + lock *sync.Mutex + client *getter.Client + computed bool + contents map[string]artifact.OCI +} + +var _ artifact.OCICollection = (*ImageTxt)(nil) + +type Option interface { + Apply(*ImageTxt) error +} + +type withIncludeSources []string + +func (o withIncludeSources) Apply(it *ImageTxt) error { + if it.IncludeSources == nil { + it.IncludeSources = make(map[string]bool) + } + for _, s := range o { + it.IncludeSources[s] = true + } + return nil +} + +func WithIncludeSources(include ...string) Option { + return withIncludeSources(include) +} + +type withExcludeSources []string + +func (o withExcludeSources) Apply(it *ImageTxt) error { + if it.ExcludeSources == nil { + it.ExcludeSources = make(map[string]bool) + } + for _, s := range o { + it.ExcludeSources[s] = true + } + return nil +} + +func WithExcludeSources(exclude ...string) Option { + return withExcludeSources(exclude) +} + +func New(ref string, opts ...Option) (*ImageTxt, error) { + it := &ImageTxt{ + Ref: ref, + + client: getter.NewClient(getter.ClientOptions{}), + lock: &sync.Mutex{}, + } + + for i, o := range opts { + if err := o.Apply(it); err != nil { + return nil, fmt.Errorf("invalid option %d: %v", i, err) + } + } + + return it, nil +} + +func (it *ImageTxt) Contents() (map[string]artifact.OCI, error) { + it.lock.Lock() + defer it.lock.Unlock() + if !it.computed { + if err := it.compute(); err != nil { + return nil, fmt.Errorf("compute OCI layout: %v", err) + } + it.computed = true + } + return it.contents, nil +} + +func (it *ImageTxt) compute() error { + // TODO - pass in logger from context + l := log.NewLogger(os.Stdout) + + it.contents = make(map[string]artifact.OCI) + + ctx := context.TODO() + + rc, err := it.client.ContentFrom(ctx, it.Ref) + if err != nil { + return fmt.Errorf("fetch image.txt ref %s: %w", it.Ref, err) + } + defer rc.Close() + + entries, err := splitImagesTxt(rc) + if err != nil { + return fmt.Errorf("parse image.txt ref %s: %v", it.Ref, err) + } + + foundSources := make(map[string]bool) + for _, e := range entries { + for s := range e.Sources { + foundSources[s] = true + } + } + + var pullAll bool + var targetSources map[string]bool + + if len(foundSources) == 0 || (len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0) { + // pull all found images + pullAll = true + + if len(foundSources) == 0 { + l.Infof("image txt file appears to have no sources; pulling all found images") + if len(it.IncludeSources) != 0 || len(it.ExcludeSources) != 0 { + l.Warnf("ImageTxt provided include or exclude sources; ignoring") + } + } else if len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0 { + l.Infof("image-sources txt file not filtered; pulling all found images") + } + } else { + // determine sources to pull + if len(it.IncludeSources) != 0 && len(it.ExcludeSources) != 0 { + l.Warnf("ImageTxt provided include and exclude sources; using only include sources") + } + + if len(it.IncludeSources) != 0 { + targetSources = it.IncludeSources + } else { + for s := range foundSources { + targetSources[s] = true + } + for s := range it.ExcludeSources { + delete(targetSources, s) + } + } + var targetSourcesArr []string + for s := range targetSources { + targetSourcesArr = append(targetSourcesArr, s) + } + l.Infof("pulling images covering sources %s", strings.Join(targetSourcesArr, ", ")) + } + + for _, e := range entries { + var matchesSourceFilter bool + if pullAll { + l.Infof("pulling image %s", e.Reference) + } else { + for s := range e.Sources { + if targetSources[s] { + matchesSourceFilter = true + l.Infof("pulling image %s (matched source %s)", e.Reference, s) + break + } + } + } + + if pullAll || matchesSourceFilter { + curImage, err := image.NewImage(e.Reference.String()) + if err != nil { + return fmt.Errorf("pull image %s: %v", e.Reference, err) + } + it.contents[e.Reference.String()] = curImage + } + } + + return nil +} + +type imageTxtEntry struct { + Reference name.Reference + Sources map[string]bool +} + +func splitImagesTxt(r io.Reader) ([]imageTxtEntry, error) { + var entries []imageTxtEntry + scanner := bufio.NewScanner(r) + for scanner.Scan() { + curEntry := imageTxtEntry{ + Sources: make(map[string]bool), + } + + lineContent := scanner.Text() + if lineContent == "" || strings.HasPrefix(lineContent, "#") { + // skip past empty and commented lines + continue + } + splitContent := strings.Split(lineContent, " ") + if len(splitContent) > 2 { + return nil, fmt.Errorf( + "invalid image.txt format: must contain only an image reference and sources separated by space; invalid line: %q", + lineContent) + } + + curRef, err := name.ParseReference(splitContent[0]) + if err != nil { + return nil, fmt.Errorf("invalid reference %s: %v", splitContent[0], err) + } + curEntry.Reference = curRef + + if len(splitContent) == 2 { + for _, source := range strings.Split(splitContent[1], ",") { + curEntry.Sources[source] = true + } + } + + entries = append(entries, curEntry) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scan contents: %v", err) + } + + return entries, nil +}