Skip to content

Commit

Permalink
Merge pull request #462 from tonistiigi/llbconstraints
Browse files Browse the repository at this point in the history
llb: force platform in llb and allow constraints
  • Loading branch information
AkihiroSuda authored Jun 25, 2018
2 parents a084629 + 242697a commit 19612b9
Show file tree
Hide file tree
Showing 13 changed files with 460 additions and 161 deletions.
54 changes: 29 additions & 25 deletions client/llb/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ type Meta struct {
ProxyEnv *ProxyEnv
}

func NewExecOp(root Output, meta Meta, readOnly bool, md OpMetadata) *ExecOp {
e := &ExecOp{meta: meta, cachedOpMetadata: md}
func NewExecOp(root Output, meta Meta, readOnly bool, c Constraints) *ExecOp {
e := &ExecOp{meta: meta, constraints: c}
rootMount := &mount{
target: pb.RootMount,
source: root,
Expand All @@ -28,10 +28,13 @@ func NewExecOp(root Output, meta Meta, readOnly bool, md OpMetadata) *ExecOp {
if readOnly {
e.root = root
} else {
e.root = &output{vertex: e, getIndex: e.getMountIndexFn(rootMount)}
o := &output{vertex: e, getIndex: e.getMountIndexFn(rootMount)}
if p := c.Platform; p != nil {
o.platform = p
}
e.root = o
}
rootMount.output = e.root

return e
}

Expand All @@ -48,13 +51,12 @@ type mount struct {
}

type ExecOp struct {
root Output
mounts []*mount
meta Meta
cachedPBDigest digest.Digest
cachedPB []byte
cachedOpMetadata OpMetadata
isValidated bool
MarshalCache
root Output
mounts []*mount
meta Meta
constraints Constraints
isValidated bool
}

func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output {
Expand All @@ -71,9 +73,13 @@ func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Outp
} else if m.tmpfs {
m.output = &output{vertex: e, err: errors.Errorf("tmpfs mount for %s can't be used as a parent", target)}
} else {
m.output = &output{vertex: e, getIndex: e.getMountIndexFn(m)}
o := &output{vertex: e, getIndex: e.getMountIndexFn(m)}
if p := e.constraints.Platform; p != nil {
o.platform = p
}
m.output = o
}
e.cachedPB = nil
e.Store(nil, nil, nil)
e.isValidated = false
return m.output
}
Expand Down Expand Up @@ -108,9 +114,9 @@ func (e *ExecOp) Validate() error {
return nil
}

func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if e.cachedPB != nil {
return e.cachedPBDigest, e.cachedPB, &e.cachedOpMetadata, nil
func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
if e.Cached(c) {
return e.Load()
}
if err := e.Validate(); err != nil {
return "", nil, nil, err
Expand Down Expand Up @@ -138,10 +144,9 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
}
}

pop := &pb.Op{
Op: &pb.Op_Exec{
Exec: peo,
},
pop, md := MarshalConstraints(c, &e.constraints)
pop.Op = &pb.Op_Exec{
Exec: peo,
}

outIndex := 0
Expand All @@ -151,7 +156,7 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if m.tmpfs {
return "", nil, nil, errors.Errorf("tmpfs mounts must use scratch")
}
inp, err := m.source.ToInput()
inp, err := m.source.ToInput(c)
if err != nil {
return "", nil, nil, err
}
Expand Down Expand Up @@ -210,9 +215,8 @@ func (e *ExecOp) Marshal() (digest.Digest, []byte, *OpMetadata, error) {
if err != nil {
return "", nil, nil, err
}
e.cachedPBDigest = digest.FromBytes(dt)
e.cachedPB = dt
return e.cachedPBDigest, dt, &e.cachedOpMetadata, nil
e.Store(dt, md, c)
return e.Load()
}

func (e *ExecOp) Output() Output {
Expand Down Expand Up @@ -376,7 +380,7 @@ func WithProxy(ps ProxyEnv) RunOption {
}

type ExecInfo struct {
opMetaWrapper
constraintsWrapper
State State
Mounts []MountInfo
ReadonlyRootFS bool
Expand Down
43 changes: 21 additions & 22 deletions client/llb/llbbuild/llbbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,20 @@ func NewBuildOp(source llb.Output, opt ...BuildOption) llb.Vertex {
for _, o := range opt {
o(info)
}
return &build{source: source, info: info, cachedOpMetadata: info.OpMetadata}
return &build{source: source, info: info, constraints: info.Constraints}
}

type build struct {
source llb.Output
info *BuildInfo
cachedPBDigest digest.Digest
cachedPB []byte
cachedOpMetadata llb.OpMetadata
llb.MarshalCache
source llb.Output
info *BuildInfo
cachedPBDigest digest.Digest
cachedPB []byte
constraints llb.Constraints
}

func (b *build) ToInput() (*pb.Input, error) {
dgst, _, _, err := b.Marshal()
func (b *build) ToInput(c *llb.Constraints) (*pb.Input, error) {
dgst, _, _, err := b.Marshal(c)
if err != nil {
return nil, err
}
Expand All @@ -44,9 +45,9 @@ func (b *build) Validate() error {
return nil
}

func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
if b.cachedPB != nil {
return b.cachedPBDigest, b.cachedPB, &b.cachedOpMetadata, nil
func (b *build) Marshal(c *llb.Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
if b.Cached(c) {
return b.Load()
}
pbo := &pb.BuildOp{
Builder: pb.LLBBuilder,
Expand All @@ -60,13 +61,12 @@ func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
pbo.Attrs[pb.AttrLLBDefinitionFilename] = b.info.DefinitionFilename
}

pop := &pb.Op{
Op: &pb.Op_Build{
Build: pbo,
},
pop, md := llb.MarshalConstraints(c, &b.constraints)
pop.Op = &pb.Op_Build{
Build: pbo,
}

inp, err := b.source.ToInput()
inp, err := b.source.ToInput(c)
if err != nil {
return "", nil, nil, err
}
Expand All @@ -77,9 +77,8 @@ func (b *build) Marshal() (digest.Digest, []byte, *llb.OpMetadata, error) {
if err != nil {
return "", nil, nil, err
}
b.cachedPB = dt
b.cachedPBDigest = digest.FromBytes(dt)
return b.cachedPBDigest, dt, &b.cachedOpMetadata, nil
b.Store(dt, md, c)
return b.Load()
}

func (b *build) Output() llb.Output {
Expand All @@ -91,7 +90,7 @@ func (b *build) Inputs() []llb.Output {
}

type BuildInfo struct {
llb.OpMetadata
llb.Constraints
DefinitionFilename string
}

Expand All @@ -103,8 +102,8 @@ func WithFilename(fn string) BuildOption {
}
}

func WithMetadata(md llb.MetadataOpt) BuildOption {
func WithConstraints(co llb.ConstraintsOpt) BuildOption {
return func(b *BuildInfo) {
md.SetMetadataOption(&b.OpMetadata)
co.SetConstraintsOption(&b.Constraints)
}
}
4 changes: 2 additions & 2 deletions client/llb/llbbuild/llbbuild_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
func TestMarshal(t *testing.T) {
t.Parallel()
b := NewBuildOp(newDummyOutput("foobar"), WithFilename("myfilename"))
dgst, dt, opMeta, err := b.Marshal()
dgst, dt, opMeta, err := b.Marshal(&llb.Constraints{})
_ = opMeta
require.NoError(t, err)

Expand Down Expand Up @@ -42,7 +42,7 @@ type dummyOutput struct {
dgst digest.Digest
}

func (d *dummyOutput) ToInput() (*pb.Input, error) {
func (d *dummyOutput) ToInput(*llb.Constraints) (*pb.Input, error) {
return &pb.Input{
Digest: d.dgst,
Index: pb.OutputIndex(7), // random constant
Expand Down
163 changes: 163 additions & 0 deletions client/llb/llbtest/platform_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
package llbtest

import (
"testing"

"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/llbsolver"
"github.com/moby/buildkit/solver/pb"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/require"
)

func TestCustomPlatform(t *testing.T) {
t.Parallel()

s := llb.Image("foo", llb.LinuxArmhf).
Run(llb.Shlex("baz")).
Run(llb.Shlex("bar")).
Run(llb.Shlex("bax"), llb.Windows).
Run(llb.Shlex("bay"))

def, err := s.Marshal()
require.NoError(t, err)

e, err := llbsolver.Load(def.ToPB())
require.NoError(t, err)

require.Equal(t, depth(e), 5)

expected := specs.Platform{OS: "windows", Architecture: "amd64"}
require.Equal(t, expected, platform(e))
e = parent(e, 0)
require.Equal(t, expected, platform(e))
e = parent(e, 0)

expected = specs.Platform{OS: "linux", Architecture: "arm", Variant: "v7"}
require.Equal(t, expected, platform(e))
e = parent(e, 0)
require.Equal(t, expected, platform(e))
require.Equal(t, []string{"baz"}, args(e))
e = parent(e, 0)
require.Equal(t, expected, platform(e))
require.Equal(t, "docker-image://docker.io/library/foo:latest", id(e))
}

func TestDefaultPlatform(t *testing.T) {
t.Parallel()

s := llb.Image("foo").Run(llb.Shlex("bar"))

def, err := s.Marshal()
require.NoError(t, err)

e, err := llbsolver.Load(def.ToPB())
require.NoError(t, err)

require.Equal(t, depth(e), 2)

expected := platforms.DefaultSpec()
require.Equal(t, expected, platform(e))
require.Equal(t, []string{"bar"}, args(e))
e = parent(e, 0)
require.Equal(t, expected, platform(e))
require.Equal(t, "docker-image://docker.io/library/foo:latest", id(e))
}

func TestPlatformOnMarshal(t *testing.T) {
t.Parallel()

s := llb.Image("image1").Run(llb.Shlex("bar"))

def, err := s.Marshal(llb.Windows)
require.NoError(t, err)

e, err := llbsolver.Load(def.ToPB())
require.NoError(t, err)

expected := specs.Platform{OS: "windows", Architecture: "amd64"}
require.Equal(t, expected, platform(e))
e = parent(e, 0)
require.Equal(t, expected, platform(e))
require.Equal(t, "docker-image://docker.io/library/image1:latest", id(e))
}

func TestPlatformMixed(t *testing.T) {
t.Parallel()

s1 := llb.Image("image1").Run(llb.Shlex("cmd-main"))
s2 := llb.Image("image2", llb.LinuxArmel).Run(llb.Shlex("cmd-sub"))
s1.AddMount("/mnt", s2.Root())

def, err := s1.Marshal(llb.LinuxAmd64)
require.NoError(t, err)

e, err := llbsolver.Load(def.ToPB())
require.NoError(t, err)

require.Equal(t, depth(e), 4)

expectedAmd := specs.Platform{OS: "linux", Architecture: "amd64"}
require.Equal(t, []string{"cmd-main"}, args(e))
require.Equal(t, expectedAmd, platform(e))

e1 := mount(e, "/")
require.Equal(t, "docker-image://docker.io/library/image1:latest", id(e1))
require.Equal(t, expectedAmd, platform(e1))

expectedArm := specs.Platform{OS: "linux", Architecture: "arm", Variant: "v6"}
e2 := mount(e, "/mnt")
require.Equal(t, []string{"cmd-sub"}, args(e2))
require.Equal(t, expectedArm, platform(e2))
e2 = parent(e2, 0)
require.Equal(t, "docker-image://docker.io/library/image2:latest", id(e2))
require.Equal(t, expectedArm, platform(e2))
}

func toOp(e solver.Edge) *pb.Op {
return e.Vertex.Sys().(*pb.Op)
}

func platform(e solver.Edge) specs.Platform {
op := toOp(e)
p := *op.Platform
return specs.Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSVersion: p.OSVersion,
OSFeatures: p.OSFeatures,
}
}

func depth(e solver.Edge) int {
i := 1
for _, inp := range e.Vertex.Inputs() {
i += depth(inp)
}
return i
}

func parent(e solver.Edge, i int) solver.Edge {
return e.Vertex.Inputs()[i]
}

func id(e solver.Edge) string {
return toOp(e).GetSource().Identifier
}

func args(e solver.Edge) []string {
return toOp(e).GetExec().Meta.Args
}

func mount(e solver.Edge, target string) solver.Edge {
op := toOp(e).GetExec()
for _, m := range op.Mounts {
if m.Dest == target {
return e.Vertex.Inputs()[int(m.Input)]
}
}
panic("could not find mount " + target)
}
Loading

0 comments on commit 19612b9

Please sign in to comment.