Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ArchGitChecksum template command in bashbrew cat #89

Merged
merged 2 commits into from
Jan 12, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add ArchGitChecksum template command in bashbrew cat
This also finally adds `bashbrew context` as an explicit subcommand so that issues with this code are easier to test/debug (so we can generate the actual tarball and compare it to previous versions of it, versions generated by `git archive`, etc).

As-is, this currently generates verbatim identical checksums to https://github.com/docker-library/meta-scripts/blob/0cde8de57dfe411ed5578feffe1b10f811e11dc2/sources.sh#L90-L96 (by design).  We'll wait to do any cache bust there until we implement `Dockerfile`/context filtering:

```console
$ bashbrew cat varnish:stable --format '{{ .TagEntry.GitCommit }} {{ .TagEntry.Directory }}'
0c295b528f28a98650fb2580eab6d34b30b165c4 stable/debian
$ git -C "$BASHBREW_CACHE/git" archive 0c295b528f28a98650fb2580eab6d34b30b165c4:stable/debian/ | ./tar-scrubber | sha256sum
3aef5ac859b23d65dfe5e9f2a47750e9a32852222829cfba762a870c1473fad6
$ bashbrew cat --format '{{ .ArchGitChecksum arch .TagEntry }}' varnish:stable
3aef5ac859b23d65dfe5e9f2a47750e9a32852222829cfba762a870c1473fad6
```

(Choosing `varnish:stable` there because it currently has [some 100% valid dangling symlinks](https://github.com/varnish/docker-varnish/tree/6b1c6ffedcfececac71e46a85122c1adaef25868/stable/debian/scripts) that tripped up my code beautifully 💕)

From a performance perspective (which was the original reason for looking into / implementing this), running the `meta-scripts/sources.sh` script against `--all` vs this, my local system gets ~18.5m vs ~4.5m (faster being this new pure-Go implementation).
  • Loading branch information
tianon committed Jan 12, 2024
commit 2d67127dd127e013915f0f516024bf1f82cdeee4
14 changes: 14 additions & 0 deletions cmd/bashbrew/git.go
Original file line number Diff line number Diff line change
@@ -96,6 +96,20 @@ func getGitCommit(commit string) (string, error) {
return h.String(), nil
}

func (r Repo) archGitFS(arch string, entry *manifest.Manifest2822Entry) (fs.FS, error) {
commit, err := r.fetchGitRepo(arch, entry)
if err != nil {
return nil, fmt.Errorf("failed fetching %q: %w", r.EntryIdentifier(entry), err)
}

gitFS, err := gitCommitFS(commit)
if err != nil {
return nil, err
}

return fs.Sub(gitFS, entry.ArchDirectory(arch))
}

func gitCommitFS(commit string) (fs.FS, error) {
if err := ensureGitInit(); err != nil {
return nil, err
57 changes: 57 additions & 0 deletions cmd/bashbrew/main.go
Original file line number Diff line number Diff line change
@@ -3,11 +3,13 @@ package main
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"

"github.com/sirupsen/logrus" // this is used by containerd libraries, so we need to set the default log level for it
"github.com/urfave/cli"
xTerm "golang.org/x/term"

"github.com/docker-library/bashbrew/architecture"
"github.com/docker-library/bashbrew/manifest"
@@ -421,6 +423,61 @@ func main() {

Category: "plumbing",
},
{
Name: "context",
Usage: "(eventually Dockerfile-filtered) git archive",
Flags: []cli.Flag{
cli.BoolFlag{
Name: "sha256",
Usage: `print sha256 instead of raw tar`,
},
// TODO "unfiltered" or something for not applying Dockerfile filtering (once that's implemented)
},
Before: subcommandBeforeFactory("context"),
Action: func(c *cli.Context) error {
repos, err := repos(false, c.Args()...)
if err != nil {
return err
}
if len(repos) != 1 {
return fmt.Errorf("'context' expects to act on exactly one architecture of one entry of one repo (got %d repos)", len(repos))
}

r, err := fetch(repos[0])
if err != nil {
return err
}

// TODO technically something like "hello-world:latest" *could* be relaxed a little if it resolves via architecture to one and only one entry 🤔 (but that's a little hard to implement with the existing internal data structures -- see TODO at the top of "sort.go")

if r.TagEntry == nil {
return fmt.Errorf("'context' expects to act on exactly one architecture of one entry of one repo (no specific entry of %q selected)", r.RepoName)
}
if len(r.TagEntries) != 1 {
return fmt.Errorf("'context' expects to act on exactly one architecture of one entry of one repo (got %d entires)", len(r.TagEntries))
}

if !r.TagEntry.HasArchitecture(arch) {
return fmt.Errorf("%q does not include architecture %q", path.Join(namespace, r.RepoName)+":"+r.TagEntry.Tags[0], arch)
}

if c.Bool("sha256") {
sum, err := r.ArchGitChecksum(arch, r.TagEntry)
if err != nil {
return err
}
fmt.Println(sum)
return nil
} else {
if xTerm.IsTerminal(int(os.Stdout.Fd())) {
return fmt.Errorf("cowardly refusing to output a tar to a terminal")
}
return r.archContextTar(arch, r.TagEntry, os.Stdout)
}
},

Category: "plumbing",
},
{
Name: "remote",
Usage: "query registries for bashbrew-related data",
1 change: 1 addition & 0 deletions cmd/bashbrew/oci-builder.go
Original file line number Diff line number Diff line change
@@ -89,6 +89,7 @@ func importOCIBlob(ctx context.Context, cs content.Store, fs iofs.FS, descriptor

// this is "docker build" but for "Builder: oci-import"
func ociImportBuild(tags []string, commit, dir, file string) (*imagespec.Descriptor, error) {
// TODO use r.archGitFS (we have no r or arch or entry here 😅)
fs, err := gitCommitFS(commit)
if err != nil {
return nil, err
10 changes: 6 additions & 4 deletions cmd/bashbrew/sort.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@ import (
"pault.ag/go/topsort"
)

// TODO unify archFilter and applyConstraints handling by pre-filtering the full list of Repo objects such that all that remains are things we should process (thus removing all "if" statements throughout the various loops); re-doing the Architectures and Entries lists to only include ones we should process, etc

func sortRepos(repos []string, applyConstraints bool) ([]string, error) {
rs := []*Repo{}
rsMap := map[*Repo]string{}
@@ -103,10 +105,10 @@ func sortRepoObjects(rs []*Repo, applyConstraints bool) ([]*Repo, error) {
continue
}
/*
// TODO need archFilter here :(
if archFilter && !entry.HasArchitecture(arch) {
continue
}
// TODO need archFilter here :(
if archFilter && !entry.HasArchitecture(arch) {
continue
}
*/

entryArches := []string{arch}
28 changes: 28 additions & 0 deletions cmd/bashbrew/tar.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package main

import (
"crypto/sha256"
"fmt"
"io"

"github.com/docker-library/bashbrew/manifest"
"github.com/docker-library/bashbrew/pkg/tarscrub"
)

func (r Repo) archContextTar(arch string, entry *manifest.Manifest2822Entry, w io.Writer) error {
f, err := r.archGitFS(arch, entry)
if err != nil {
return err
}

return tarscrub.WriteTar(f, w)
}

func (r Repo) ArchGitChecksum(arch string, entry *manifest.Manifest2822Entry) (string, error) {
h := sha256.New()
err := r.archContextTar(arch, entry, h)
if err != nil {
return "", err
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ require (
github.com/sirupsen/logrus v1.9.0
github.com/urfave/cli v1.22.10
go.etcd.io/bbolt v1.3.7
golang.org/x/term v0.5.0
pault.ag/go/debian v0.12.0
pault.ag/go/topsort v0.1.1
)
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -940,6 +940,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Loading