buildinfo: handle deps

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

client/client_test.go

@@ -5198,7 +5198,7 @@ func testBuildInfoInline(t *testing.T, sb integration.Sandbox) {
 			require.NoError(t, err)
 
 			var bi binfotypes.BuildInfo
-			err = json.Unmarshal(config.BuildInfo, &bi)
+			err = json.Unmarshal([]byte(config.BuildInfo), &bi)
 			require.NoError(t, err)
 
 			if tt.buildAttrs {

docs/build-repro.md

@@ -6,18 +6,6 @@ Build dependencies are generated when your image has been built. These
 dependencies include versions of used images, git repositories and HTTP URLs
 used by LLB `Source` operation as well as build request attributes.
 
-By default, the build dependencies are inlined in the image configuration. You
-can disable this behavior with the [`buildinfo` attribute](../README.md#imageregistry).
-
-### Image config
-
-A new field similar to the one for inline cache has been added to the image
-configuration to embed build dependencies:
-
-```text
-"moby.buildkit.buildinfo.v1": <base64>
-```
-
 The structure is base64 encoded and has the following format when decoded:
 
 ```json
@@ -57,10 +45,25 @@ The structure is base64 encoded and has the following format when decoded:
 
 * `frontend` defines the frontend used to build.
 * `attrs` defines build request attributes.
-* `sources` defines build dependencies.
+* `sources` defines build sources.
   * `type` defines the source type (`docker-image`, `git` or `http`).
   * `ref` is the reference of the source.
   * `pin` is the source digest.
+* `deps` defines underlying dependencies.
+
+### Image config
+
+A new field similar to the one for inline cache has been added to the image
+configuration to embed build dependencies:
+
+```json
+{
+  "moby.buildkit.buildinfo.v0": "<base64>"
+}
+```
+
+By default, the build dependencies are inlined in the image configuration. You
+can disable this behavior with the [`buildinfo` attribute](../README.md#imageregistry).
 
 ### Exporter response (metadata)
 

frontend/dockerfile/dockerfile2llb/convert.go

@@ -25,6 +25,7 @@ import (
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
+	"github.com/moby/buildkit/util/buildinfo"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/suggest"
 	"github.com/moby/buildkit/util/system"
@@ -355,6 +356,15 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 								Alias: ref.String(),
 								Pin:   dgst.String(),
 							}
+							// also set dependencies if source image contains buildinfo.
+							// use build source reference as key
+							if srcbuildinfo, err := buildinfo.FromImageConfig(dt); err != nil {
+								return err
+							} else if srcbuildinfo != nil {
+								d.sourceDeps = map[string]*binfotypes.BuildInfo{
+									origName: srcbuildinfo,
+								}
+							}
 						}
 						d.image = img
 					}
@@ -382,16 +392,26 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 
 	buildContext := &mutableOutput{}
 	ctxPaths := map[string]struct{}{}
-	buildinfo := &binfotypes.BuildInfo{}
+	buildInfo := &binfotypes.BuildInfo{}
 
 	for _, d := range allDispatchStates.states {
 		if !isReachable(target, d) {
 			continue
 		}
 
-		// collect build dependencies
+		// collect build sources and dependencies
 		if d.buildSource != nil {
-			buildinfo.Sources = append(buildinfo.Sources, *d.buildSource)
+			buildInfo.Sources = append(buildInfo.Sources, *d.buildSource)
+		}
+		if d.sourceDeps != nil && len(d.sourceDeps) > 0 {
+			if buildInfo.Deps == nil {
+				buildInfo.Deps = make(map[string]binfotypes.BuildInfo)
+			}
+			for k, dep := range d.sourceDeps {
+				if dep != nil {
+					buildInfo.Deps[k] = *dep
+				}
+			}
 		}
 
 		if d.base != nil {
@@ -469,9 +489,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	}
 
 	// sort build sources
-	if len(buildinfo.Sources) > 0 {
-		sort.Slice(buildinfo.Sources, func(i, j int) bool {
-			return buildinfo.Sources[i].Ref < buildinfo.Sources[j].Ref
+	if len(buildInfo.Sources) > 0 {
+		sort.Slice(buildInfo.Sources, func(i, j int) bool {
+			return buildInfo.Sources[i].Ref < buildInfo.Sources[j].Ref
 		})
 	}
 
@@ -512,7 +532,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		target.image.Variant = platformOpt.targetPlatform.Variant
 	}
 
-	return &st, &target.image, buildinfo, nil
+	return &st, &target.image, buildInfo, nil
 }
 
 func metaArgsToMap(metaArgs []instructions.KeyValuePairOptional) map[string]string {
@@ -702,6 +722,7 @@ type dispatchState struct {
 	cmdTotal       int
 	prefixPlatform bool
 	buildSource    *binfotypes.Source
+	sourceDeps     map[string]*binfotypes.BuildInfo
 }
 
 type dispatchStates struct {

util/buildinfo/buildinfo.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/source"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/urlutil"
@@ -32,15 +33,18 @@ func Encode(ctx context.Context, buildInfo []byte, buildSources map[string]strin
 			return nil, err
 		}
 	}
-	msources, err := mergeSources(ctx, buildSources, bi.Sources)
-	if err != nil {
+	if deps, err := decodeDeps(bi.Attrs); err == nil {
+		bi.Deps = reduceMapBuildInfo(deps, bi.Deps)
+	} else {
 		return nil, err
 	}
-	return json.Marshal(binfotypes.BuildInfo{
-		Frontend: bi.Frontend,
-		Attrs:    filterAttrs(bi.Attrs),
-		Sources:  msources,
-	})
+	if sources, err := mergeSources(ctx, buildSources, bi.Sources); err == nil {
+		bi.Sources = sources
+	} else {
+		return nil, err
+	}
+	bi.Attrs = filterAttrs(bi.Attrs)
+	return json.Marshal(bi)
 }
 
 // mergeSources combines and fixes build sources from frontend sources.
@@ -136,6 +140,32 @@ func mergeSources(ctx context.Context, buildSources map[string]string, frontendS
 	return srcs, nil
 }
 
+func decodeDeps(attrs map[string]*string) (map[string]binfotypes.BuildInfo, error) {
+	res := make(map[string]binfotypes.BuildInfo)
+	for k, v := range attrs {
+		if v == nil || !strings.HasPrefix(k, "input-metadata:") {
+			continue
+		}
+		var inputresp map[string]string
+		if err := json.Unmarshal([]byte(*v), &inputresp); err != nil {
+			return nil, errors.Wrap(err, "failed to unmarshal input-metadata")
+		}
+		if _, ok := inputresp[exptypes.ExporterBuildInfo]; !ok {
+			continue
+		}
+		bi, err := Decode(inputresp[exptypes.ExporterBuildInfo])
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to decode build info from input-metadata")
+		}
+		kl := strings.SplitN(k, ":", 2)
+		res[kl[1]] = bi
+	}
+	if len(res) == 0 {
+		return nil, nil
+	}
+	return res, nil
+}
+
 // FormatOpts holds build info format options.
 type FormatOpts struct {
 	RemoveAttrs bool
@@ -239,15 +269,25 @@ func GetMetadata(metadata map[string][]byte, key string, reqFrontend string, req
 		if reqFrontend != "" {
 			mbi.Frontend = reqFrontend
 		}
-		mbi.Attrs = filterAttrs(convertMap(reduceMap(reqAttrs, mbi.Attrs)))
+		if deps, err := decodeDeps(convertMap(reduceMapString(reqAttrs, mbi.Attrs))); err == nil {
+			mbi.Deps = reduceMapBuildInfo(deps, mbi.Deps)
+		} else {
+			return nil, err
+		}
+		mbi.Attrs = filterAttrs(convertMap(reduceMapString(reqAttrs, mbi.Attrs)))
 		dtbi, err = json.Marshal(mbi)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to marshal build info for %q", key)
 		}
 	} else {
+		deps, err := decodeDeps(convertMap(reqAttrs))
+		if err != nil {
+			return nil, err
+		}
 		dtbi, err = json.Marshal(binfotypes.BuildInfo{
 			Frontend: reqFrontend,
 			Attrs:    filterAttrs(convertMap(reqAttrs)),
+			Deps:     deps,
 		})
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to marshal build info for %q", key)
@@ -256,7 +296,26 @@ func GetMetadata(metadata map[string][]byte, key string, reqFrontend string, req
 	return dtbi, nil
 }
 
-func reduceMap(m1 map[string]string, m2 map[string]*string) map[string]string {
+// FromImageConfig returns build info from image config.
+func FromImageConfig(dt []byte) (*binfotypes.BuildInfo, error) {
+	if len(dt) == 0 {
+		return nil, nil
+	}
+	var config binfotypes.ImageConfig
+	if err := json.Unmarshal(dt, &config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal image config")
+	}
+	if len(config.BuildInfo) == 0 {
+		return nil, nil
+	}
+	bi, err := Decode(config.BuildInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode build info from image config")
+	}
+	return &bi, nil
+}
+
+func reduceMapString(m1 map[string]string, m2 map[string]*string) map[string]string {
 	if m1 == nil && m2 == nil {
 		return nil
 	}
@@ -271,6 +330,19 @@ func reduceMap(m1 map[string]string, m2 map[string]*string) map[string]string {
 	return m1
 }
 
+func reduceMapBuildInfo(m1 map[string]binfotypes.BuildInfo, m2 map[string]binfotypes.BuildInfo) map[string]binfotypes.BuildInfo {
+	if m1 == nil && m2 == nil {
+		return nil
+	}
+	if m1 == nil {
+		m1 = map[string]binfotypes.BuildInfo{}
+	}
+	for k, v := range m2 {
+		m1[k] = v
+	}
+	return m1
+}
+
 func convertMap(m map[string]string) map[string]*string {
 	res := make(map[string]*string)
 	for k, v := range m {

util/buildinfo/buildinfo_test.go

@@ -91,6 +91,34 @@ func TestMergeSources(t *testing.T) {
 	}, srcs)
 }
 
+func TestDecodeDeps(t *testing.T) {
+	deps, err := decodeDeps(map[string]*string{
+		"build-arg:bar":         stringPtr("foo"),
+		"build-arg:foo":         stringPtr("bar"),
+		"context:baseapp":       stringPtr("input:0-base"),
+		"filename":              stringPtr("Dockerfile"),
+		"input-metadata:0-base": stringPtr("{\"containerimage.buildinfo\":\"eyJmcm9udGVuZCI6ImRvY2tlcmZpbGUudjAiLCJhdHRycyI6eyJidWlsZC1hcmc6YmFyIjoiZm9vIiwiYnVpbGQtYXJnOmZvbyI6ImJhciIsImZpbGVuYW1lIjoiYmFzZWFwcC5Eb2NrZXJmaWxlIn0sInNvdXJjZXMiOlt7InR5cGUiOiJkb2NrZXItaW1hZ2UiLCJyZWYiOiJidXN5Ym94IiwiYWxpYXMiOiJkb2NrZXIuaW8vbGlicmFyeS9idXN5Ym94QHNoYTI1NjphZmNjN2YxYWMxYjQ5ZGIzMTdhNzE5NmM5MDJlNjFjNmMzYzQ2MDdkNjM1OTllZTFhODJkNzAyZDI0OWEwY2NiIiwicGluIjoic2hhMjU2OmFmY2M3ZjFhYzFiNDlkYjMxN2E3MTk2YzkwMmU2MWM2YzNjNDYwN2Q2MzU5OWVlMWE4MmQ3MDJkMjQ5YTBjY2IifV19\",\"containerimage.config\":\"eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsIm9zIjoibGludXgiLCJyb290ZnMiOnsidHlwZSI6ImxheWVycyIsImRpZmZfaWRzIjpbInNoYTI1NjpkMzE1MDVmZDUwNTBmNmI5NmNhMzI2OGQxZGI1OGZjOTFhZTU2MWRkZjE0ZWFhYmM0MWQ2M2VhMmVmOGMxYzZkIl19LCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMi0wMi0wNFQyMToyMDoxMi4zMTg5MTc4MjJaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBmaWxlOjFjODUwN2UzZTliMjJiOTc3OGYyZWRiYjk1MDA2MWUwNmJkZTZhMWY1M2I2OWUxYzYxMDI1MDAyOWMzNzNiNzIgaW4gLyAifSx7ImNyZWF0ZWQiOiIyMDIyLTAyLTA0VDIxOjIwOjEyLjQ5Nzc5NDgwOVoiLCJjcmVhdGVkX2J5IjoiL2Jpbi9zaCAtYyAjKG5vcCkgIENNRCBbXCJzaFwiXSIsImVtcHR5X2xheWVyIjp0cnVlfSx7ImNyZWF0ZWRfYnkiOiJXT1JLRElSIC9zcmMiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9XSwiY29uZmlnIjp7IkVudiI6WyJQQVRIPS91c3IvbG9jYWwvc2JpbjovdXNyL2xvY2FsL2JpbjovdXNyL3NiaW46L3Vzci9iaW46L3NiaW46L2JpbiJdLCJDbWQiOlsic2giXSwiV29ya2luZ0RpciI6Ii9zcmMiLCJPbkJ1aWxkIjpudWxsfX0=\"}"),
+	})
+	require.NoError(t, err)
+	require.Contains(t, deps, "0-base")
+	require.Equal(t, binfotypes.BuildInfo{
+		Frontend: "dockerfile.v0",
+		Attrs: map[string]*string{
+			"build-arg:bar": stringPtr("foo"),
+			"build-arg:foo": stringPtr("bar"),
+			"filename":      stringPtr("baseapp.Dockerfile"),
+		},
+		Sources: []binfotypes.Source{
+			{
+				Type:  binfotypes.SourceTypeDockerImage,
+				Ref:   "busybox",
+				Alias: "docker.io/library/busybox@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb",
+				Pin:   "sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb",
+			},
+		},
+	}, deps["0-base"])
+}
+
 func TestFormat(t *testing.T) {
 	bi := binfotypes.BuildInfo{
 		Frontend: "dockerfile.v0",
@@ -193,7 +221,7 @@ func TestReduceMap(t *testing.T) {
 	for _, tt := range cases {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			require.Equal(t, tt.expected, reduceMap(tt.m2, tt.m1))
+			require.Equal(t, tt.expected, reduceMapString(tt.m2, tt.m1))
 		})
 	}
 }

util/buildinfo/types/types.go

@@ -10,7 +10,7 @@ const ImageConfigField = "moby.buildkit.buildinfo.v1"
 // ImageConfig defines the structure of build dependencies
 // inside image config.
 type ImageConfig struct {
-	BuildInfo []byte `json:"moby.buildkit.buildinfo.v1,omitempty"`
+	BuildInfo string `json:"moby.buildkit.buildinfo.v1,omitempty"`
 }
 
 // BuildInfo defines the main structure added to image config as
@@ -23,6 +23,8 @@ type BuildInfo struct {
 	Attrs map[string]*string `json:"attrs,omitempty"`
 	// Sources defines build dependencies.
 	Sources []Source `json:"sources,omitempty"`
+	// Deps defines context dependencies.
+	Deps map[string]BuildInfo `json:"deps,omitempty"`
 }
 
 // Source defines a build dependency.