diff --git a/.gitignore b/.gitignore index d897571..144c59d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +/oci-cas /oci-create-runtime-bundle -/oci-unpack +/oci-image-init /oci-image-validate +/oci-refs +/oci-unpack diff --git a/.tool/lint b/.tool/lint index 4a0120e..bcbcda7 100755 --- a/.tool/lint +++ b/.tool/lint @@ -14,6 +14,12 @@ for d in $(find . -type d -not -iwholename '*.git*' -a -not -iname '.tool' -a -n --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \ --exclude='.*_test\.go:.*error return value not checked.*\(errcheck\)$' \ --exclude='duplicate of.*_test.go.*\(dupl\)$' \ + --exclude='^cmd/oci-cas/delete.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-cas/get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-refs/delete.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-refs/get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^image/cas/layout/dir.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^image/refs/layout/dir.go:.* duplicate of .* \(dupl\)$' \ --exclude='schema/fs.go' \ --exclude='duplicate of.*main.go.*\(dupl\)$' \ --disable=aligncheck \ diff --git a/Makefile b/Makefile index f0f35f7..c58a4b4 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,11 @@ COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true) EPOCH_TEST_COMMIT ?= v0.2.0 TOOLS := \ + oci-cas \ oci-create-runtime-bundle \ + oci-image-init \ oci-image-validate \ + oci-refs \ oci-unpack default: help diff --git a/cmd/oci-cas/delete.go b/cmd/oci-cas/delete.go new file mode 100644 index 0000000..f6c2c25 --- /dev/null +++ b/cmd/oci-cas/delete.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type deleteCmd struct { + path string + digest string +} + +func newDeleteCmd() *cobra.Command { + state := &deleteCmd{} + + return &cobra.Command{ + Use: "delete PATH DIGEST", + Short: "Remove a blob from from the store", + Run: state.Run, + } +} + +func (state *deleteCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.digest = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *deleteCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.Delete(ctx, state.digest) +} diff --git a/cmd/oci-cas/get.go b/cmd/oci-cas/get.go new file mode 100644 index 0000000..3a7d139 --- /dev/null +++ b/cmd/oci-cas/get.go @@ -0,0 +1,93 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type getCmd struct { + path string + digest string +} + +func newGetCmd() *cobra.Command { + state := &getCmd{} + + return &cobra.Command{ + Use: "get PATH DIGEST", + Short: "Retrieve a blob from the store", + Long: "Retrieve a blob from the store and write it to stdout.", + Run: state.Run, + } +} + +func (state *getCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.digest = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *getCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + reader, err := engine.Get(ctx, state.digest) + if err != nil { + return err + } + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + + n, err := os.Stdout.Write(bytes) + if err != nil { + return err + } + if n < len(bytes) { + return fmt.Errorf("wrote %d of %d bytes", n, len(bytes)) + } + + return nil +} diff --git a/cmd/oci-cas/main.go b/cmd/oci-cas/main.go new file mode 100644 index 0000000..59bff68 --- /dev/null +++ b/cmd/oci-cas/main.go @@ -0,0 +1,39 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-cas", + Short: "Content-addressable storage manipulation", + } + + cmd.AddCommand(newGetCmd()) + cmd.AddCommand(newPutCmd()) + cmd.AddCommand(newDeleteCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/oci-cas/oci-cas-delete.1.md b/cmd/oci-cas/oci-cas-delete.1.md new file mode 100644 index 0000000..09d444f --- /dev/null +++ b/cmd/oci-cas/oci-cas-delete.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% SEPTEMBER 2016 +# NAME + +oci-cas-get \- Remove a blob from the store + +# SYNOPSIS + +**oci-cas delete** [OPTIONS] PATH DIGEST + +# DESCRIPTION + +`oci-cas delete` removes the blob referenced by `DIGEST` from the store at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-get**(1), **oci-cas-put**(1) + +# HISTORY + +September 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas-get.1.md b/cmd/oci-cas/oci-cas-get.1.md new file mode 100644 index 0000000..3d55466 --- /dev/null +++ b/cmd/oci-cas/oci-cas-get.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas-get \- Retrieve a blob from the store + +# SYNOPSIS + +**oci-cas get** [OPTIONS] PATH DIGEST + +# DESCRIPTION + +`oci-cas get` retrieves a blob referenced by `DIGEST` from the store at `PATH` and writes it to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-put**(1), **oci-cas-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas-put.1.md b/cmd/oci-cas/oci-cas-put.1.md new file mode 100644 index 0000000..ffe867b --- /dev/null +++ b/cmd/oci-cas/oci-cas-put.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas-put \- Write a blob to the store + +# SYNOPSIS + +**oci-cas put** [OPTIONS] PATH + +# DESCRIPTION + +`oci-cas put` reads a blob from stdin, writes it to the store at `PATH`, and prints the digest to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-get**(1), **oci-cas-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas.1.md b/cmd/oci-cas/oci-cas.1.md new file mode 100644 index 0000000..9c8912d --- /dev/null +++ b/cmd/oci-cas/oci-cas.1.md @@ -0,0 +1,52 @@ +% OCI(1) OCI-User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas \- Content-addressable storage manipulation + +# SYNOPSIS + +**oci-cas** [command] + +# DESCRIPTION + +`oci-cas` manipulates content-addressable storage. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**get** + Retrieve a blob from the store. + See **oci-cas-get**(1) for full documentation on the **get** command. + +**put** + Write a blob to the store. + See **oci-cas-put**(1) for full documentation on the **put** command. + +**delete** + Remove a blob from the store. + See **oci-cas-delete**(1) for full documentation on the **delete** command. + +# EXAMPLES + +``` +$ oci-image-init image-layout image +$ echo hello | oci-cas put image +sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +$ oci-cas get image sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +hello +$ oci-cas delete image sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +``` + +# SEE ALSO + +**oci-image-tools**(7), **oci-cas-get**(1), **oci-cas-put**(1), **oci-cas-delete**(1), **oci-image-init**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/put.go b/cmd/oci-cas/put.go new file mode 100644 index 0000000..31abd2b --- /dev/null +++ b/cmd/oci-cas/put.go @@ -0,0 +1,83 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type putCmd struct { + path string +} + +func newPutCmd() *cobra.Command { + state := &putCmd{} + + return &cobra.Command{ + Use: "put PATH", + Short: "Write a blob to the store", + Long: "Read a blob from stdin, write it to the store, and print the digest to stdout.", + Run: state.Run, + } +} + +func (state *putCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *putCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + digest, err := engine.Put(ctx, os.Stdin) + if err != nil { + return err + } + + n, err := fmt.Fprintln(os.Stdout, digest) + if err != nil { + return err + } + if n < len(digest) { + return fmt.Errorf("wrote %d of %d bytes", n, len(digest)) + } + + return nil +} diff --git a/cmd/oci-create-runtime-bundle/main.go b/cmd/oci-create-runtime-bundle/main.go index c517d09..f76563c 100644 --- a/cmd/oci-create-runtime-bundle/main.go +++ b/cmd/oci-create-runtime-bundle/main.go @@ -23,6 +23,7 @@ import ( specs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -31,7 +32,6 @@ var gitCommit = "" // supported bundle types var bundleTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -109,6 +109,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if _, err := os.Stat(args[1]); os.IsNotExist(err) { v.stderr.Printf("destination path %s does not exist", args[1]) os.Exit(1) @@ -125,11 +127,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.CreateRuntimeBundleLayout(args[0], args[1], v.ref, v.root) - case image.TypeImage: - err = image.CreateRuntimeBundle(args[0], args[1], v.ref, v.root) + err = image.CreateRuntimeBundle(ctx, args[0], args[1], v.ref, v.root) } if err != nil { diff --git a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md index ad95181..cd3223d 100644 --- a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md +++ b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md @@ -24,7 +24,7 @@ runtime-spec-compatible `dest/config.json`. A directory representing the root filesystem of the container in the OCI runtime bundle. It is strongly recommended to keep the default value. (default "rootfs") **--type** - Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "image" # EXAMPLES ``` @@ -36,7 +36,7 @@ $ cd busybox-bundle && sudo runc run busybox ``` # SEE ALSO -**runc**(1), **skopeo**(1) +**oci-image-tools**(7), **runc**(1), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-image-init/image_layout.go b/cmd/oci-image-init/image_layout.go new file mode 100644 index 0000000..f245541 --- /dev/null +++ b/cmd/oci-image-init/image_layout.go @@ -0,0 +1,76 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type imageLayout struct { + backend string +} + +func newImageLayoutCmd() *cobra.Command { + state := &imageLayout{ + backend: "dir", + } + + cmd := &cobra.Command{ + Use: "image-layout PATH", + Short: "Initialize an OCI image-layout repository", + Run: state.Run, + } + + cmd.Flags().StringVar( + &state.backend, "type", "dir", + "Select the image-backend. Choices: dir, tar. Defaults to dir.", + ) + + return cmd +} + +func (state *imageLayout) Run(cmd *cobra.Command, args []string) { + var err error + if len(args) != 1 { + if err = cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + path := args[0] + + ctx := context.Background() + + switch state.backend { + case "dir": + err = layout.CreateDir(ctx, path) + case "tar": + err = layout.CreateTarFile(ctx, path) + default: + err = fmt.Errorf("unrecognized type: %q", state.backend) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/oci-image-init/main.go b/cmd/oci-image-init/main.go new file mode 100644 index 0000000..49f4aab --- /dev/null +++ b/cmd/oci-image-init/main.go @@ -0,0 +1,37 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-image-init", + Short: "Initialize an OCI image", + } + + cmd.AddCommand(newImageLayoutCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/oci-image-init/oci-image-init-image-layout.1.md b/cmd/oci-image-init/oci-image-init-image-layout.1.md new file mode 100644 index 0000000..3351bbf --- /dev/null +++ b/cmd/oci-image-init/oci-image-init-image-layout.1.md @@ -0,0 +1,32 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-image-init-image-layout \- Initialize an OCI image-layout repository + +# SYNOPSIS + +**oci-image-init image-layout** [OPTIONS] PATH + +# DESCRIPTION + +`oci-image-init image-layout` initializes an image-layout repository at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +**--type** + Select the image-layout backend. + Choices: dir, tar. + Defaults to dir. + +# SEE ALSO + +**oci-image-init**(1), **oci-cas**(1), ***oci-refs**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-init/oci-image-init.1.md b/cmd/oci-image-init/oci-image-init.1.md new file mode 100644 index 0000000..3f1eaa9 --- /dev/null +++ b/cmd/oci-image-init/oci-image-init.1.md @@ -0,0 +1,33 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-image-init \- Initialize an OCI image + +# SYNOPSIS + +**oci-image-init** [command] + +# DESCRIPTION + +`oci-image-init` Initializes an OCI image. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**image-layout** + Initialize an OCI image-layout repository. + See **oci-image-init-image-layout**(1) for full documentation on the **image-layout** command. + +# SEE ALSO + +**oci-image-tool**(1), **oci-image-init-image-layout**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tools.7.md b/cmd/oci-image-tools.7.md new file mode 100644 index 0000000..f9c0754 --- /dev/null +++ b/cmd/oci-image-tools.7.md @@ -0,0 +1,40 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% JULY 2016 +# NAME + +oci-image-tools \- OCI (Open Container Initiative) image tools + +# DESCRIPTION + +The OCI image tools are a collection of tools for working with the OCI image specification. + +# COMMANDS + +**oci-cas**(1) + Content-addressable storage manipulation. + +**oci-create-runtime-bundle**(1) + Create an OCI image runtime bundle + +**oci-image-init**(1) + Initialize an OCI image. + +**oci-refs**(1) + Name-based reference manipulation. + +**oci-unpack**(1) + Unpack an image or image source layout + +**oci-validate**(1) + Validate one or more image files + +# SEE ALSO + +**skopeo**(1), +https://github.com/opencontainers/image-spec, +https://github.com/opencontainers/image-tools + +# HISTORY + +July 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-image-validate/main.go b/cmd/oci-image-validate/main.go index 5143b63..0fc0978 100644 --- a/cmd/oci-image-validate/main.go +++ b/cmd/oci-image-validate/main.go @@ -25,6 +25,7 @@ import ( "github.com/opencontainers/image-tools/image" "github.com/pkg/errors" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -33,7 +34,6 @@ var gitCommit = "" // supported validation types var validateTypes = []string{ - image.TypeImageLayout, image.TypeImage, image.TypeManifest, image.TypeManifestList, @@ -81,7 +81,7 @@ func newValidateCmd(stdout, stderr *log.Logger) *cobra.Command { cmd.Flags().StringSliceVar( &v.refs, "ref", nil, - `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout.`, + `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image.`, ) cmd.Flags().BoolVar( @@ -107,9 +107,11 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + var exitcode int for _, arg := range args { - err := v.validatePath(arg) + err := v.validatePath(ctx, arg) if err == nil { v.stdout.Printf("%s: OK", arg) @@ -139,7 +141,7 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(exitcode) } -func (v *validateCmd) validatePath(name string) error { +func (v *validateCmd) validatePath(ctx context.Context, name string) error { var ( err error typ = v.typ @@ -152,10 +154,8 @@ func (v *validateCmd) validatePath(name string) error { } switch typ { - case image.TypeImageLayout: - return image.ValidateLayout(name, v.refs, v.stdout) case image.TypeImage: - return image.Validate(name, v.refs, v.stdout) + return image.Validate(ctx, name, v.refs, v.stdout) } f, err := os.Open(name) diff --git a/cmd/oci-image-validate/oci-image-validate.1.md b/cmd/oci-image-validate/oci-image-validate.1.md index 33d2b49..cec1250 100644 --- a/cmd/oci-image-validate/oci-image-validate.1.md +++ b/cmd/oci-image-validate/oci-image-validate.1.md @@ -20,20 +20,20 @@ oci-image-validate \- Validate one or more image files Can be specified multiple times to validate multiple references. `NAME` must be present in the `refs` subdirectory of the image. Defaults to `v1.0`. - Only applicable if type is image or imageLayout. + Only applicable if type is image. **--type** - Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "imageLayout,image,manifest,manifestList,config" + Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "image,manifest,manifestList,config" # EXAMPLES ``` $ skopeo copy docker://busybox oci:busybox-oci -$ oci-image-validate --type imageLayout --ref latest busybox-oci +$ oci-image-validate --type image --ref latest busybox-oci busybox-oci: OK ``` # SEE ALSO -**skopeo**(1) +**oci-image-tools**(7), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-refs/delete.go b/cmd/oci-refs/delete.go new file mode 100644 index 0000000..0e661cd --- /dev/null +++ b/cmd/oci-refs/delete.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type deleteCmd struct { + path string + name string +} + +func newDeleteCmd() *cobra.Command { + state := &deleteCmd{} + + return &cobra.Command{ + Use: "delete PATH NAME", + Short: "Remove a reference from the store", + Run: state.Run, + } +} + +func (state *deleteCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and NAME must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *deleteCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.Delete(ctx, state.name) +} diff --git a/cmd/oci-refs/get.go b/cmd/oci-refs/get.go new file mode 100644 index 0000000..5daf671 --- /dev/null +++ b/cmd/oci-refs/get.go @@ -0,0 +1,78 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type getCmd struct { + path string + name string +} + +func newGetCmd() *cobra.Command { + state := &getCmd{} + + return &cobra.Command{ + Use: "get PATH NAME", + Short: "Retrieve a reference from the store", + Run: state.Run, + } +} + +func (state *getCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and NAME must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *getCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + descriptor, err := engine.Get(ctx, state.name) + if err != nil { + return err + } + + return json.NewEncoder(os.Stdout).Encode(&descriptor) +} diff --git a/cmd/oci-refs/list.go b/cmd/oci-refs/list.go new file mode 100644 index 0000000..4e5e6d3 --- /dev/null +++ b/cmd/oci-refs/list.go @@ -0,0 +1,78 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type listCmd struct { + path string +} + +func newListCmd() *cobra.Command { + state := &listCmd{} + + return &cobra.Command{ + Use: "list PATH", + Short: "Return available names from the store.", + Run: state.Run, + } +} + +func (state *listCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "PATH must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *listCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.List(ctx, "", -1, 0, state.printName) +} + +func (state *listCmd) printName(ctx context.Context, name string) (err error) { + n, err := fmt.Fprintln(os.Stdout, name) + if n < len(name) { + return fmt.Errorf("wrote %d of %d name", n, len(name)) + } + return err +} diff --git a/cmd/oci-refs/main.go b/cmd/oci-refs/main.go new file mode 100644 index 0000000..e7a96cb --- /dev/null +++ b/cmd/oci-refs/main.go @@ -0,0 +1,40 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-refs", + Short: "Name-based reference manipulation", + } + + cmd.AddCommand(newPutCmd()) + cmd.AddCommand(newGetCmd()) + cmd.AddCommand(newListCmd()) + cmd.AddCommand(newDeleteCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/oci-refs/oci-refs-delete.1.md b/cmd/oci-refs/oci-refs-delete.1.md new file mode 100644 index 0000000..b477743 --- /dev/null +++ b/cmd/oci-refs/oci-refs-delete.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% SEPTEMBER 2016 +# NAME + +oci-refs-delete \- Remove a reference from the store + +# SYNOPSIS + +**oci-refs delete** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs delete` removes reference `NAME` from the store at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1) + +# HISTORY + +September 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-get.1.md b/cmd/oci-refs/oci-refs-get.1.md new file mode 100644 index 0000000..49afd8d --- /dev/null +++ b/cmd/oci-refs/oci-refs-get.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-get \- Retrieve a reference from the store + +# SYNOPSIS + +**oci-refs get** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs get` retrieves reference `NAME` from the store at `PATH` and writes the JSON descriptor to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-list**(1), **oci-refs-put**(1), **oci-refs-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-list.1.md b/cmd/oci-refs/oci-refs-list.1.md new file mode 100644 index 0000000..38c610b --- /dev/null +++ b/cmd/oci-refs/oci-refs-list.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-list \- Return available names from the store + +# SYNOPSIS + +**oci-refs list** [OPTIONS] PATH + +# DESCRIPTION + +`oci-refs list` retrieves all names from the store at `PATH` and writes them to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-put**(1), **oci-refs-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-put.1.md b/cmd/oci-refs/oci-refs-put.1.md new file mode 100644 index 0000000..61c5098 --- /dev/null +++ b/cmd/oci-refs/oci-refs-put.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-put \- Write a reference to the store + +# SYNOPSIS + +**oci-refs put** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs put` reads descriptor JSON from standard input and writes it to the store at `PATH` as `NAME`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs.1.md b/cmd/oci-refs/oci-refs.1.md new file mode 100644 index 0000000..5f9a284 --- /dev/null +++ b/cmd/oci-refs/oci-refs.1.md @@ -0,0 +1,60 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs \- Name-based reference manipulation + +# SYNOPSIS + +**oci-refs** [command] + +# DESCRIPTION + +`oci-refs` manipulates name-based references. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**get** + Retrieve a reference from the store. + See **oci-refs-get**(1) for full documentation on the **get** command. + +**list** + Return available names from the store. + See **oci-refs-list**(1) for full documentation on the **list** command. + +**put** + Write a reference to the store. + See **oci-refs-put**(1) for full documentation on the **put** command. + +**delete** + Remove a reference from the store. + See **oci-refs-delete**(1) for full documentation on the **delete** command. + +# EXAMPLES + +``` +$ oci-image-init image-layout image +$ DIGEST=$(echo hello | oci-cas put image) +$ SIZE=$(echo hello | wc -c) +$ printf '{"mediaType": "text/plain", "digest": "%s", "size": %d}' "${DIGEST}" "${SIZE}" | +> oci-refs put image greeting +$ oci-refs list image +greeting +$ oci-refs get image greeting +{"mediaType":"text/plain","digest":"sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03","size":6} +$ oci-refs delete image greeting +``` + +# SEE ALSO + +**oci-image-tools**(7), **oci-cas-put**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1), **oci-refs-delete**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/put.go b/cmd/oci-refs/put.go new file mode 100644 index 0000000..5d35e37 --- /dev/null +++ b/cmd/oci-refs/put.go @@ -0,0 +1,81 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type putCmd struct { + path string + name string +} + +func newPutCmd() *cobra.Command { + state := &putCmd{} + + return &cobra.Command{ + Use: "put PATH NAME", + Short: "Write a reference to the store", + Long: "Read descriptor JSON from stdin and write it to the store.", + Run: state.Run, + } +} + +func (state *putCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *putCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + decoder := json.NewDecoder(os.Stdin) + var descriptor specs.Descriptor + err = decoder.Decode(&descriptor) + if err != nil { + return err + } + + return engine.Put(ctx, state.name, &descriptor) +} diff --git a/cmd/oci-unpack/main.go b/cmd/oci-unpack/main.go index ed0ffa7..e754d50 100644 --- a/cmd/oci-unpack/main.go +++ b/cmd/oci-unpack/main.go @@ -23,6 +23,7 @@ import ( specs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -31,7 +32,6 @@ var gitCommit = "" // supported unpack types var unpackTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -62,8 +62,8 @@ func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command { cmd := &cobra.Command{ Use: "unpack [src] [dest]", - Short: "Unpack an image or image source layout", - Long: `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`, + Short: "Unpack an image", + Long: `Unpack the OCI image present at [src] to the destination directory [dest].`, Run: v.Run, } @@ -101,6 +101,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if v.typ == "" { typ, err := image.Autodetect(args[0]) if err != nil { @@ -112,11 +114,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.UnpackLayout(args[0], args[1], v.ref) - case image.TypeImage: - err = image.Unpack(args[0], args[1], v.ref) + err = image.Unpack(ctx, args[0], args[1], v.ref) } if err != nil { diff --git a/cmd/oci-unpack/oci-unpack.1.md b/cmd/oci-unpack/oci-unpack.1.md index b7529f9..33f2854 100644 --- a/cmd/oci-unpack/oci-unpack.1.md +++ b/cmd/oci-unpack/oci-unpack.1.md @@ -18,7 +18,7 @@ oci-unpack \- Unpack an image or image source layout The ref pointing to the manifest to be unpacked. This must be present in the "refs" subdirectory of the image. (default "v1.0") **--type** - Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "image" # EXAMPLES ``` @@ -43,7 +43,7 @@ busybox-bundle ``` # SEE ALSO -**skopeo**(1) +**oci-image-tools**(7), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/glide.lock b/glide.lock index bffc373..3301bc4 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 14550151b754f92de80e864f26da93e8adfce71537c9cb7883b0bdd67e541453 -updated: 2016-09-15T10:25:30.038113538+02:00 +updated: 2016-09-16T00:10:45.625507345-07:00 imports: - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 @@ -29,4 +29,8 @@ imports: version: e7a2449258501866491620d4f47472e6100ca551 subpackages: - errorutil +- name: golang.org/x/net + version: 71a035914f99bb58fe82eac0f1289f10963d876c + subpackages: + - context testImports: [] diff --git a/image/autodetect.go b/image/autodetect.go index c3e2f85..5b0552e 100644 --- a/image/autodetect.go +++ b/image/autodetect.go @@ -27,7 +27,6 @@ import ( // supported autodetection types const ( - TypeImageLayout = "imageLayout" TypeImage = "image" TypeManifest = "manifest" TypeManifestList = "manifestList" @@ -43,7 +42,7 @@ func Autodetect(path string) (string, error) { } if fi.IsDir() { - return TypeImageLayout, nil + return TypeImage, nil } f, err := os.Open(path) diff --git a/image/cas/interface.go b/image/cas/interface.go new file mode 100644 index 0000000..58df8dd --- /dev/null +++ b/image/cas/interface.go @@ -0,0 +1,48 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cas implements generic content-addressable storage. +package cas + +import ( + "io" + + "golang.org/x/net/context" +) + +// Engine represents a content-addressable storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. +type Engine interface { + + // Put adds a new blob to the store. The action is idempotent; a + // nil return means "that content is stored at DIGEST" without + // implying "because of your Put()". + Put(ctx context.Context, reader io.Reader) (digest string, err error) + + // Get returns a reader for retrieving a blob from the store. + // Returns os.ErrNotExist if the digest is not found. + Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) + + // Delete removes a blob from the store. The action is idempotent; a + // nil return means "that content is not in the store" without + // implying "because of your Delete()". + Delete(ctx context.Context, digest string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/cas/layout/dir.go b/image/cas/layout/dir.go new file mode 100644 index 0000000..7e729c6 --- /dev/null +++ b/image/cas/layout/dir.go @@ -0,0 +1,122 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "crypto/sha256" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/opencontainers/image-tools/image/cas" + "github.com/opencontainers/image-tools/image/layout" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// DirEngine is a cas.Engine backed by a directory. +type DirEngine struct { + path string + temp string +} + +// NewDirEngine returns a new DirEngine. +func NewDirEngine(ctx context.Context, path string) (eng cas.Engine, err error) { + engine := &DirEngine{ + path: path, + } + + err = layout.CheckDirVersion(ctx, engine.path) + if err != nil { + return nil, err + } + + tempDir, err := ioutil.TempDir(path, "tmp-") + if err != nil { + return nil, err + } + engine.temp = tempDir + + return engine, nil +} + +// Put adds a new blob to the store. +func (engine *DirEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { + hash := sha256.New() + algorithm := "sha256" + + var file *os.File + file, err = ioutil.TempFile(engine.temp, "blob-") + if err != nil { + return "", err + } + defer func() { + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + err = errors.Wrap(err, err2.Error()) + } + } + }() + defer file.Close() + + hashingWriter := io.MultiWriter(file, hash) + _, err = io.Copy(hashingWriter, reader) + if err != nil { + return "", err + } + file.Close() + + digest = fmt.Sprintf("%s:%x", algorithm, hash.Sum(nil)) + targetName, err := blobPath(digest, string(os.PathSeparator)) + if err != nil { + return "", err + } + + path := filepath.Join(engine.path, targetName) + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return "", err + } + + err = os.Rename(file.Name(), path) + if err != nil { + return "", err + } + + return digest, nil +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *DirEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) { + targetName, err := blobPath(digest, string(os.PathSeparator)) + if err != nil { + return nil, err + } + + return os.Open(filepath.Join(engine.path, targetName)) +} + +// Delete removes a blob from the store. +func (engine *DirEngine) Delete(ctx context.Context, digest string) (err error) { + return layout.DirDelete(ctx, engine.path, digest, blobPath) +} + +// Close releases resources held by the engine. +func (engine *DirEngine) Close() (err error) { + return os.RemoveAll(engine.temp) +} diff --git a/image/cas/layout/interface.go b/image/cas/layout/interface.go new file mode 100644 index 0000000..dda72d6 --- /dev/null +++ b/image/cas/layout/interface.go @@ -0,0 +1,25 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" +) + +// ReadWriteSeekCloser wraps the Read, Write, Seek, and Close methods. +type ReadWriteSeekCloser interface { + io.ReadWriteSeeker + io.Closer +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go new file mode 100644 index 0000000..de9ab9e --- /dev/null +++ b/image/cas/layout/main.go @@ -0,0 +1,57 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the cas interface using the image-spec's +// image-layout [1]. +// +// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "fmt" + "os" + "strings" + + "github.com/opencontainers/image-tools/image/cas" + "golang.org/x/net/context" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { + engine, err = NewDirEngine(ctx, path) + if err == nil { + return engine, err + } + + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err == nil { + return NewTarEngine(ctx, file) + } + + return nil, fmt.Errorf("unrecognized engine at %q", path) +} + +// blobPath returns the PATH to the DIGEST blob. SEPARATOR selects +// the path separator used between components. +func blobPath(digest string, separator string) (path string, err error) { + fields := strings.SplitN(digest, ":", 2) + if len(fields) != 2 { + return "", fmt.Errorf("invalid digest: %q, %v", digest, fields) + } + algorithm := fields[0] + hash := fields[1] + components := []string{".", "blobs", algorithm, hash} + return strings.Join(components, separator), nil +} diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go new file mode 100644 index 0000000..d8269e9 --- /dev/null +++ b/image/cas/layout/tar.go @@ -0,0 +1,108 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + + "github.com/opencontainers/image-tools/image/cas" + "github.com/opencontainers/image-tools/image/layout" + "golang.org/x/net/context" +) + +// TarEngine is a cas.Engine backed by a tar file. +type TarEngine struct { + file ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (eng cas.Engine, err error) { + engine := &TarEngine{ + file: file, + } + + err = layout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + + return engine, nil +} + +// Put adds a new blob to the store. +func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { + data, err := ioutil.ReadAll(reader) + if err != nil { + return "", err + } + + size := int64(len(data)) + hash := sha256.Sum256(data) + hexHash := hex.EncodeToString(hash[:]) + algorithm := "sha256" + digest = fmt.Sprintf("%s:%s", algorithm, hexHash) + + _, err = engine.Get(ctx, digest) + if err == os.ErrNotExist { + var targetName string + targetName, err = blobPath(digest, "/") + if err != nil { + return "", err + } + + reader = bytes.NewReader(data) + err = layout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) + if err != nil { + return "", err + } + } else if err != nil { + return "", err + } + + return digest, nil +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) { + targetName, err := blobPath(digest, "/") + if err != nil { + return nil, err + } + + _, tarReader, err := layout.TarEntryByName(ctx, engine.file, targetName) + if err != nil { + return nil, err + } + + return ioutil.NopCloser(tarReader), nil +} + +// Delete removes a blob from the store. +func (engine *TarEngine) Delete(ctx context.Context, digest string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} diff --git a/image/cas/put.go b/image/cas/put.go new file mode 100644 index 0000000..0f61881 --- /dev/null +++ b/image/cas/put.go @@ -0,0 +1,47 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cas + +import ( + "bytes" + "encoding/json" + + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// PutJSON writes a generic JSON object to content-addressable storage +// and returns a Descriptor referencing it. +func PutJSON(ctx context.Context, engine Engine, data interface{}, mediaType string) (descriptor *specs.Descriptor, err error) { + jsonBytes, err := json.Marshal(data) + if err != nil { + return nil, err + } + size := len(jsonBytes) + size64 := int64(size) // panics on overflow + + reader := bytes.NewReader(jsonBytes) + digest, err := engine.Put(ctx, reader) + if err != nil { + return nil, err + } + + descriptor = &specs.Descriptor{ + MediaType: mediaType, + Digest: digest, + Size: size64, + } + return descriptor, nil +} diff --git a/image/config.go b/image/config.go index 100d99c..81daea9 100644 --- a/image/config.go +++ b/image/config.go @@ -18,63 +18,64 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" - "os" - "path/filepath" "strconv" "strings" "github.com/opencontainers/image-spec/schema" + imagespecs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" + runtimespecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type config v1.Image +func findConfig(ctx context.Context, engine cas.Engine, descriptor *imagespecs.Descriptor) (config *v1.Image, err error) { + err = validateMediaType(descriptor.MediaType, []string{v1.MediaTypeImageConfig}) + if err != nil { + return nil, errors.Wrap(err, "invalid config media type") + } -func findConfig(w walker, d *descriptor) (*config, error) { - var c config - cpath := filepath.Join("blobs", d.algo(), d.hash()) + err = validateDescriptor(ctx, engine, descriptor) + if err != nil { + return nil, errors.Wrap(err, "invalid config descriptor") + } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != cpath { - return nil - } - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading config", path) - } + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: config validation failed", path) - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if err := json.Unmarshal(buf, &c); err != nil { - return err - } - // check if the rootfs type is 'layers' - if c.RootFS.Type != "layers" { - return fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type) - } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: config not found", cpath) - case errEOW: - return &c, nil - default: + if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: config validation failed", descriptor.Digest) + } + + var c v1.Image + if err := json.Unmarshal(buf, &c); err != nil { return nil, err } + + // check if the rootfs type is 'layers' + if c.RootFS.Type != "layers" { + return nil, fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type) + } + + return &c, nil } -func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { +func runtimeSpec(c *v1.Image, rootfs string) (*runtimespecs.Spec, error) { if c.OS != "linux" { return nil, fmt.Errorf("%s: unsupported OS", c.OS) } - var s specs.Spec - s.Version = specs.Version + var s runtimespecs.Spec + s.Version = runtimespecs.Version // we should at least apply the default spec, otherwise this is totally useless s.Process.Terminal = true s.Root.Path = rootfs @@ -116,12 +117,12 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { swap := uint64(c.Config.MemorySwap) shares := uint64(c.Config.CPUShares) - s.Linux.Resources = &specs.Resources{ - CPU: &specs.CPU{ + s.Linux.Resources = &runtimespecs.Resources{ + CPU: &runtimespecs.CPU{ Shares: &shares, }, - Memory: &specs.Memory{ + Memory: &runtimespecs.Memory{ Limit: &mem, Reservation: &mem, Swap: &swap, @@ -131,7 +132,7 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { for vol := range c.Config.Volumes { s.Mounts = append( s.Mounts, - specs.Mount{ + runtimespecs.Mount{ Destination: vol, Type: "bind", Options: []string{"rbind"}, diff --git a/image/descriptor.go b/image/descriptor.go index 1e760a7..73eaf89 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -17,105 +17,34 @@ package image import ( "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" - "os" - "path/filepath" - "strings" + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func (d *descriptor) algo() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[0] -} - -func (d *descriptor) hash() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[1] -} - -func listReferences(w walker) (map[string]*descriptor, error) { - refs := make(map[string]*descriptor) - - if err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || !strings.HasPrefix(path, "refs") { +func validateMediaType(mediaType string, mediaTypes []string) error { + for _, mt := range mediaTypes { + if mt == mediaType { return nil } - - var d descriptor - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - refs[info.Name()] = &d - - return nil - }); err != nil { - return nil, err } - return refs, nil + return fmt.Errorf("invalid media type %q", mediaType) } -func findDescriptor(w walker, name string) (*descriptor, error) { - var d descriptor - dpath := filepath.Join("refs", name) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != dpath { - return nil - } - - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: descriptor not found", dpath) - case errEOW: - return &d, nil - default: - return nil, err - } -} - -func (d *descriptor) validate(w walker, mts []string) error { - var found bool - for _, mt := range mts { - if d.MediaType == mt { - found = true - break - } - } - if !found { - return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType) - } - - rc, err := w.Get(*d) +func validateDescriptor(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) error { + reader, err := engine.Get(ctx, descriptor.Digest) if err != nil { - return err + return errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) } - defer rc.Close() - return d.validateContent(rc) + return validateContent(ctx, descriptor, reader) } -func (d *descriptor) validateContent(r io.Reader) error { +func validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error { h := sha256.New() n, err := io.Copy(h, r) if err != nil { @@ -124,11 +53,11 @@ func (d *descriptor) validateContent(r io.Reader) error { digest := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if digest != d.Digest { + if digest != descriptor.Digest { return errors.New("digest mismatch") } - if n != d.Size { + if n != descriptor.Size { return errors.New("size mismatch") } diff --git a/image/image.go b/image/image.go index 1551c1c..70d7019 100644 --- a/image/image.go +++ b/image/image.go @@ -16,176 +16,203 @@ package image import ( "encoding/json" - "fmt" "log" "os" "path/filepath" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/opencontainers/image-tools/image/refs" + refslayout "github.com/opencontainers/image-tools/image/refs/layout" "github.com/pkg/errors" + "golang.org/x/net/context" ) -// ValidateLayout walks through the given file tree and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func ValidateLayout(src string, refs []string, out *log.Logger) error { - return validate(newPathWalker(src), refs, out) +var validRefMediaTypes = []string{ + v1.MediaTypeImageManifest, + v1.MediaTypeImageManifestList, } -// Validate walks through the given .tar file and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func Validate(tarFile string, refs []string, out *log.Logger) error { - f, err := os.Open(tarFile) +// Validate validates the given reference. +func Validate(ctx context.Context, path string, refs []string, out *log.Logger) error { + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer refEngine.Close() - return validate(newTarWalker(tarFile, f), refs, out) -} + casEngine, err := caslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer casEngine.Close() -var validRefMediaTypes = []string{ - v1.MediaTypeImageManifest, - v1.MediaTypeImageManifestList, -} + if len(refs) > 0 { + for _, ref := range refs { + err = validate(ctx, refEngine, casEngine, ref, out) + if err != nil { + return err + } + } + } -func validate(w walker, refs []string, out *log.Logger) error { - ds, err := listReferences(w) + count := 0 + err = refEngine.List( + ctx, + "", + -1, + 0, + func(ctx context.Context, name string) error { + count++ + return validate(ctx, refEngine, casEngine, name, out) + }, + ) if err != nil { return err } - if len(refs) == 0 && len(ds) == 0 { + + if count == 0 { // TODO(runcom): ugly, we'll need a better way and library // to express log levels. // see https://github.com/opencontainers/image-spec/issues/288 out.Print("WARNING: no descriptors found") } - if len(refs) == 0 { - for ref := range ds { - refs = append(refs, ref) - } + return nil +} + +func validate(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, ref string, out *log.Logger) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) } - for _, ref := range refs { - d, ok := ds[ref] - if !ok { - // TODO(runcom): - // soften this error to a warning if the user didn't ask for any specific reference - // with --ref but she's just validating the whole image. - return fmt.Errorf("reference %s not found", ref) - } + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) + if err != nil { + return err + } - if err = d.validate(w, validRefMediaTypes); err != nil { - return err - } + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { + return err + } - m, err := findManifest(w, d) - if err != nil { - return err - } + m, err := findManifest(ctx, casEngine, descriptor) + if err != nil { + return err + } - if err := m.validate(w); err != nil { - return err - } - if out != nil { - out.Printf("reference %q: OK", ref) - } + err = validateManifest(ctx, m, casEngine) + if err != nil { + return err + } + + if out != nil { + out.Printf("reference %q: OK", ref) } return nil } -// UnpackLayout walks through the file tree given by src and, using the layers -// specified in the manifest pointed to by the given ref, unpacks all layers in -// the given destination directory or returns an error if the unpacking failed. -func UnpackLayout(src, dest, ref string) error { - return unpack(newPathWalker(src), dest, ref) -} +// Unpack unpacks the given reference to a destination directory. +func Unpack(ctx context.Context, path, dest, ref string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// Unpack walks through the given .tar file and, using the layers specified in -// the manifest pointed to by the given ref, unpacks all layers in the given -// destination directory or returns an error if the unpacking failed. -func Unpack(tarFile, dest, ref string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return unpack(newTarWalker(tarFile, f), dest, ref) + return unpack(ctx, refEngine, casEngine, dest, ref) } -func unpack(w walker, dest, refName string) error { - ref, err := findDescriptor(w, refName) +func unpack(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - return m.unpack(w, dest) + return unpackManifest(ctx, m, casEngine, dest) } -// CreateRuntimeBundleLayout walks through the file tree given by src and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundleLayout(src, dest, ref, root string) error { - return createRuntimeBundle(newPathWalker(src), dest, ref, root) -} +// CreateRuntimeBundle creates an OCI runtime bundle in the given +// destination. +func CreateRuntimeBundle(ctx context.Context, path, dest, ref, rootfs string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// CreateRuntimeBundle walks through the given .tar file and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundle(tarFile, dest, ref, root string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return createRuntimeBundle(newTarWalker(tarFile, f), dest, ref, root) + return createRuntimeBundle(ctx, refEngine, casEngine, dest, ref, rootfs) } -func createRuntimeBundle(w walker, dest, refName, rootfs string) error { - ref, err := findDescriptor(w, refName) +func createRuntimeBundle(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref, rootfs string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - c, err := findConfig(w, &m.Config) + c, err := findConfig(ctx, casEngine, &m.Config) if err != nil { return err } - err = m.unpack(w, filepath.Join(dest, rootfs)) + err = unpackManifest(ctx, m, casEngine, filepath.Join(dest, rootfs)) if err != nil { return err } - spec, err := c.runtimeSpec(rootfs) + spec, err := runtimeSpec(c, rootfs) if err != nil { return err } diff --git a/image/image_test.go b/image/image_test.go index 8c3c0c9..fec077d 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -18,24 +18,25 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "fmt" "io" "io/ioutil" "os" - "path/filepath" "strconv" "strings" "testing" + "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" + cas "github.com/opencontainers/image-tools/image/cas" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + refslayout "github.com/opencontainers/image-tools/image/refs/layout" + "golang.org/x/net/context" ) const ( refTag = "latest" - layoutStr = `{"imageLayoutVersion": "1.0.0"}` - configStr = `{ "created": "2015-10-31T22:22:56.015925234Z", "author": "Alyssa P. Hacker ", @@ -91,8 +92,6 @@ const ( ) var ( - refStr = `{"digest":"","mediaType":"application/vnd.oci.image.manifest.v1+json","size":}` - manifestStr = `{ "annotations": null, "config": { @@ -118,214 +117,106 @@ type tarContent struct { b []byte } -type imageLayout struct { - rootDir string - layout string - ref string - manifest string - config string - tarList []tarContent -} - func TestValidateLayout(t *testing.T) { + ctx := context.Background() + root, err := ioutil.TempDir("", "oci-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(root) - il := imageLayout{ - rootDir: root, - layout: layoutStr, - ref: refTag, - manifest: manifestStr, - config: configStr, - tarList: []tarContent{ - tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, - }, - } - - // create image layout bundle - err = createImageLayoutBundle(il) + err = imagelayout.CreateDir(ctx, root) if err != nil { t.Fatal(err) } - err = ValidateLayout(root, []string{refTag}, nil) + casEngine, err := caslayout.NewEngine(ctx, root) if err != nil { t.Fatal(err) } -} - -func createImageLayoutBundle(il imageLayout) error { - err := os.MkdirAll(filepath.Join(il.rootDir, "blobs", "sha256"), 0700) - if err != nil { - return err - } - - err = os.MkdirAll(filepath.Join(il.rootDir, "refs"), 0700) - if err != nil { - return err - } - - // create image layout file - err = createLayoutFile(il.rootDir) - if err != nil { - return err - } - - // create image layer blob file. - desc, err := createImageLayerFile(il.rootDir, il.tarList) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - desc, err = createConfigFile(il.rootDir, il.config) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - // create manifest blob file - desc, err = createManifestFile(il.rootDir, il.manifest) - if err != nil { - return err - } - - return createRefFile(il.rootDir, il.ref, desc) -} + defer casEngine.Close() -func createLayoutFile(root string) error { - layoutPath := filepath.Join(root, "oci-layout") - f, err := os.Create(layoutPath) + refsEngine, err := refslayout.NewEngine(ctx, root) if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(layoutStr))) - return err -} - -func createRefFile(root, ref string, mft descriptor) error { - refpath := filepath.Join(root, "refs", ref) - f, err := os.Create(refpath) - if err != nil { - return err + t.Fatal(err) } - defer f.Close() - refStr = strings.Replace(refStr, "", mft.Digest, -1) - refStr = strings.Replace(refStr, "", strconv.FormatInt(mft.Size, 10), -1) - _, err = io.Copy(f, bytes.NewBuffer([]byte(refStr))) - return err -} + defer refsEngine.Close() -func createManifestFile(root, str string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-manifest") - f, err := os.Create(name) + layer, err := createTarBlob(ctx, casEngine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + }) if err != nil { - return descriptor{}, err + t.Fatal(err) } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(str))) - if err != nil { - return descriptor{}, err + digest, err := casEngine.Put(ctx, strings.NewReader(configStr)) + config := specs.Descriptor{ + Digest: digest, + MediaType: v1.MediaTypeImageConfig, + Size: int64(len(configStr)), } - return createHashedBlob(name) -} - -func createConfigFile(root, config string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-config") - f, err := os.Create(name) - if err != nil { - return descriptor{}, err + _manifest := manifestStr + _manifest = strings.Replace(_manifest, "", config.Digest, 1) + _manifest = strings.Replace(_manifest, "", strconv.FormatInt(config.Size, 10), 1) + _manifest = strings.Replace(_manifest, "", layer.Digest, 1) + _manifest = strings.Replace(_manifest, "", strconv.FormatInt(layer.Size, 10), 1) + digest, err = casEngine.Put(ctx, strings.NewReader(_manifest)) + manifest := specs.Descriptor{ + Digest: digest, + MediaType: v1.MediaTypeImageManifest, + Size: int64(len(_manifest)), } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(config))) + err = refsEngine.Put(ctx, refTag, &manifest) if err != nil { - return descriptor{}, err - } - - return createHashedBlob(name) -} - -func createImageLayerFile(root string, list []tarContent) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-layer") - err := createTarBlob(name, list) - if err != nil { - return descriptor{}, err + t.Fatal(err) } - desc, err := createHashedBlob(name) + err = Validate(ctx, root, []string{refTag}, nil) if err != nil { - return descriptor{}, err + t.Fatal(err) } - - desc.MediaType = v1.MediaTypeImageLayer - return desc, nil } -func createTarBlob(name string, list []tarContent) error { - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - gzipWriter := gzip.NewWriter(file) +func createTarBlob(ctx context.Context, engine cas.Engine, list []tarContent) (descriptor *specs.Descriptor, err error) { + var buffer bytes.Buffer + gzipWriter := gzip.NewWriter(&buffer) defer gzipWriter.Close() tarWriter := tar.NewWriter(gzipWriter) defer tarWriter.Close() for _, content := range list { if err = tarWriter.WriteHeader(content.header); err != nil { - return err + return nil, err } if _, err = io.Copy(tarWriter, bytes.NewReader(content.b)); err != nil { - return err + return nil, err } } - return nil -} -func createHashedBlob(name string) (descriptor, error) { - desc, err := newDescriptor(name) + err = tarWriter.Close() if err != nil { - return descriptor{}, err + return nil, err } - // Rename the file to hashed-digest name. - err = os.Rename(name, filepath.Join(filepath.Dir(name), desc.Digest)) + err = gzipWriter.Close() if err != nil { - return descriptor{}, err + return nil, err } - //Normalize the hashed digest. - desc.Digest = "sha256:" + desc.Digest - - return desc, nil -} - -func newDescriptor(name string) (descriptor, error) { - file, err := os.Open(name) - if err != nil { - return descriptor{}, err + var desc = specs.Descriptor{ + MediaType: v1.MediaTypeImageLayer, + Size: int64(buffer.Len()), } - defer file.Close() - // generate sha256 hash - hash := sha256.New() - size, err := io.Copy(hash, file) + digest, err := engine.Put(ctx, &buffer) if err != nil { - return descriptor{}, err + return nil, err } - return descriptor{ - Digest: fmt.Sprintf("%x", hash.Sum(nil)), - Size: size, - }, nil + desc.Digest = digest + + return &desc, nil } diff --git a/image/layout/dir.go b/image/layout/dir.go new file mode 100644 index 0000000..a46fb54 --- /dev/null +++ b/image/layout/dir.go @@ -0,0 +1,98 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/net/context" +) + +// CheckDirVersion checks the oci-layout entry in an image-layout +// directory and returns an error if oci-layout is missing or has +// unrecognized content. +func CheckDirVersion(ctx context.Context, path string) (err error) { + file, err := os.Open(filepath.Join(path, "oci-layout")) + if os.IsNotExist(err) { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + defer file.Close() + + return CheckVersion(ctx, file) +} + +// CreateDir creates a new image-layout directory at the given path. +func CreateDir(ctx context.Context, path string) (err error) { + err = os.MkdirAll(path, 0777) + if err != nil { + return err + } + + file, err := os.OpenFile( + filepath.Join(path, "oci-layout"), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0666, + ) + if err != nil { + return err + } + defer file.Close() + + imageLayoutVersion := ImageLayoutVersion{ + Version: "1.0.0", + } + imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion) + if err != nil { + return err + } + n, err := file.Write(imageLayoutVersionBytes) + if err != nil { + return err + } + if n < len(imageLayoutVersionBytes) { + return fmt.Errorf("wrote %d of %d bytes", n, len(imageLayoutVersionBytes)) + } + + err = os.MkdirAll(filepath.Join(path, "blobs"), 0777) + if err != nil { + return err + } + + return os.MkdirAll(filepath.Join(path, "refs"), 0777) +} + +// DirDelete removes an entry from a directory, wrapping a +// path-constructing call to entryPath and a removing call to +// os.Remove. Deletion is idempotent (unlike os.Remove, where +// attempting to delete a nonexistent path results in an error). +func DirDelete(ctx context.Context, path string, entry string, entryPath EntryPath) (err error) { + targetName, err := entryPath(entry, string(os.PathSeparator)) + if err != nil { + return err + } + + err = os.Remove(filepath.Join(path, targetName)) + if os.IsNotExist(err) { + return nil + } + return err +} diff --git a/image/layout/layout.go b/image/layout/layout.go new file mode 100644 index 0000000..cf1ba7b --- /dev/null +++ b/image/layout/layout.go @@ -0,0 +1,50 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout defines utility code shared by refs/layout and cas/layout. +package layout + +import ( + "encoding/json" + "fmt" + "io" + + "golang.org/x/net/context" +) + +// EntryPath is a template for helpers that convert from ref or blob +// names to image-layout paths. +type EntryPath func(entry string, separator string) (path string, err error) + +// ImageLayoutVersion represents the oci-version content for the image +// layout format. +type ImageLayoutVersion struct { + Version string `json:"imageLayoutVersion"` +} + +// CheckVersion checks an oci-layout reader and returns an error if it +// has unrecognized content. +func CheckVersion(ctx context.Context, reader io.Reader) (err error) { + decoder := json.NewDecoder(reader) + var version ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil +} diff --git a/image/layout/tar.go b/image/layout/tar.go new file mode 100644 index 0000000..81f5a7f --- /dev/null +++ b/image/layout/tar.go @@ -0,0 +1,257 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "time" + + "golang.org/x/net/context" +) + +// TarEntryByName walks a tarball pointed to by reader, finds an +// entry matching the given name, and returns the header and reader +// for that entry. Returns os.ErrNotExist if the path is not found. +func TarEntryByName(ctx context.Context, reader io.ReadSeeker, name string) (header *tar.Header, tarReader *tar.Reader, err error) { + _, err = reader.Seek(0, os.SEEK_SET) + if err != nil { + return nil, nil, err + } + + tarReader = tar.NewReader(reader) + for { + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return nil, nil, os.ErrNotExist + } + if err != nil { + return nil, nil, err + } + + if header.Name == name { + return header, tarReader, nil + } + } +} + +// WriteTarEntryByName reads content from reader into an entry at name +// in the tarball at file, replacing a previous entry with that name +// (if any). The current implementation avoids writing a temporary +// file to disk, but risks leaving a corrupted tarball if the program +// crashes mid-write. +// +// To add an entry to a tarball (with Go's interface) you need to know +// the size ahead of time. If you set the size argument, +// WriteTarEntryByName will use that size in the entry header (and +// Go's implementation will check to make sure it matches the length +// of content read from reader). If unset, WriteTarEntryByName will +// copy reader into a local buffer, measure its size, and then write +// the entry header and content. +func WriteTarEntryByName(ctx context.Context, file io.ReadWriteSeeker, name string, reader io.Reader, size *int64) (err error) { + var buffer bytes.Buffer + tarWriter := tar.NewWriter(&buffer) + + components := strings.Split(name, "/") + if components[0] != "." { + return fmt.Errorf("tar name entry does not start with './': %q", name) + } + + var parents []string + for i := 2; i < len(components); i++ { + parents = append(parents, strings.Join(components[:i], "/")) + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + + tarReader := tar.NewReader(file) + found := false + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + dirName := strings.TrimRight(header.Name, "/") + for i, parent := range parents { + if dirName == parent { + parents = append(parents[:i], parents[i+1:]...) + break + } + } + + if header.Name == name { + found = true + err = writeTarEntry(ctx, tarWriter, name, reader, size) + } else { + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = io.Copy(tarWriter, tarReader) + } + if err != nil { + return err + } + } + + if !found { + now := time.Now() + for _, parent := range parents { + header := &tar.Header{ + Name: parent + "/", + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + err = writeTarEntry(ctx, tarWriter, name, reader, size) + if err != nil { + return err + } + } + + err = tarWriter.Close() + if err != nil { + return err + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + // FIXME: truncate file + + _, err = buffer.WriteTo(file) + return err +} + +func writeTarEntry(ctx context.Context, writer *tar.Writer, name string, reader io.Reader, size *int64) (err error) { + if size == nil { + var data []byte + data, err = ioutil.ReadAll(reader) + if err != nil { + return err + } + reader = bytes.NewReader(data) + _size := int64(len(data)) + size = &_size + } + now := time.Now() + header := &tar.Header{ + Name: name, + Mode: 0666, + Size: *size, + ModTime: now, + Typeflag: tar.TypeReg, + } + err = writer.WriteHeader(header) + if err != nil { + return err + } + + _, err = io.Copy(writer, reader) + return err +} + +// CheckTarVersion walks a tarball pointed to by reader and returns an +// error if oci-layout is missing or has unrecognized content. +func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { + _, tarReader, err := TarEntryByName(ctx, reader, "./oci-layout") + if err == os.ErrNotExist { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + + return CheckVersion(ctx, tarReader) +} + +// CreateTarFile creates a new image-layout tar file at the given path. +func CreateTarFile(ctx context.Context, path string) (err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return err + } + defer file.Close() + + tarWriter := tar.NewWriter(file) + defer tarWriter.Close() + + now := time.Now() + for _, name := range []string{"./blobs/", "./refs/"} { + header := &tar.Header{ + Name: name, + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + + imageLayoutVersion := ImageLayoutVersion{ + Version: "1.0.0", + } + imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion) + if err != nil { + return err + } + header := &tar.Header{ + Name: "./oci-layout", + Mode: 0666, + Size: int64(len(imageLayoutVersionBytes)), + ModTime: now, + Typeflag: tar.TypeReg, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = tarWriter.Write(imageLayoutVersionBytes) + return err +} diff --git a/image/manifest.go b/image/manifest.go index c438ede..fb41b04 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -28,67 +28,67 @@ import ( "time" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type manifest struct { - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` -} - -func findManifest(w walker, d *descriptor) (*manifest, error) { - var m manifest - mpath := filepath.Join("blobs", d.algo(), d.hash()) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != mpath { - return nil - } - - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading manifest", path) - } - - if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: manifest validation failed", path) - } +func findManifest(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) (*v1.Manifest, error) { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := json.Unmarshal(buf, &m); err != nil { - return err - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if len(m.Layers) == 0 { - return fmt.Errorf("%s: no layers found", path) - } + if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: manifest validation failed", descriptor.Digest) + } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: manifest not found", mpath) - case errEOW: - return &m, nil - default: + var m v1.Manifest + if err := json.Unmarshal(buf, &m); err != nil { return nil, err } + + if len(m.Layers) == 0 { + return nil, fmt.Errorf("%s: no layers found", descriptor.Digest) + } + + return &m, nil } -func (m *manifest) validate(w walker) error { - if err := m.Config.validate(w, []string{v1.MediaTypeImageConfig}); err != nil { - return errors.Wrap(err, "config validation failed") +func validateManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine) error { + _, err := findConfig(ctx, engine, &m.Config) + if err != nil { + return errors.Wrap(err, "invalid manifest config") } for _, d := range m.Layers { - if err := d.validate(w, []string{v1.MediaTypeImageLayer}); err != nil { - return errors.Wrap(err, "layer validation failed") + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") + } + err = validateDescriptor(ctx, engine, &d) + if err != nil { + return errors.Wrap(err, "invalid layer descriptor") } } return nil } -func (m *manifest) unpack(w walker, dest string) (retErr error) { +func unpackManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine, dest string) (err error) { // error out if the dest directory is not empty s, err := ioutil.ReadDir(dest) if err != nil && !os.IsNotExist(err) { @@ -100,38 +100,33 @@ func (m *manifest) unpack(w walker, dest string) (retErr error) { defer func() { // if we encounter error during unpacking // clean up the partially-unpacked destination - if retErr != nil { - if err := os.RemoveAll(dest); err != nil { + if err != nil { + err2 := os.RemoveAll(dest) + if err2 != nil { fmt.Printf("Error: failed to remove partially-unpacked destination %v", err) } } }() + for _, d := range m.Layers { - if d.MediaType != string(schema.MediaTypeImageLayer) { - continue + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - dd, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != dd { - return nil - } - - if err := unpackLayer(dest, r); err != nil { - return errors.Wrap(err, "error extracting layer") - } + reader, err := engine.Get(ctx, d.Digest) + if err != nil { + return errors.Wrapf(err, "failed to fetch %s", d.Digest) + } - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: layer not found", dest) - case errEOW: - default: - return err + if err := unpackLayer(dest, reader); err != nil { + return errors.Wrap(err, "error extracting layer") } } return nil diff --git a/image/manifest_test.go b/image/manifest_test.go index f50b2ed..64c98e4 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -18,14 +18,18 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "testing" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-spec/specs-go/v1" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + "golang.org/x/net/context" ) func TestUnpackLayerDuplicateEntries(t *testing.T) { @@ -66,53 +70,38 @@ func TestUnpackLayerDuplicateEntries(t *testing.T) { } func TestUnpackLayer(t *testing.T) { + ctx := context.Background() + tmp1, err := ioutil.TempDir("", "test-layer") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp1) - err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700) - if err != nil { - t.Fatal(err) - } - tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar") - f, err := os.Create(tarfile) - if err != nil { - t.Fatal(err) - } - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test"))) - tw.Close() - gw.Close() - f.Close() - // generate sha256 hash - h := sha256.New() - file, err := os.Open(tarfile) + path := filepath.Join(tmp1, "image.tar") + err = imagelayout.CreateDir(ctx, path) if err != nil { t.Fatal(err) } - defer file.Close() - _, err = io.Copy(h, file) + + engine, err := caslayout.NewEngine(ctx, path) if err != nil { t.Fatal(err) } - err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil)))) + defer engine.Close() + + layer, err := createTarBlob(ctx, engine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + }) if err != nil { t.Fatal(err) } - testManifest := manifest{ - Layers: []descriptor{descriptor{ - MediaType: "application/vnd.oci.image.layer.tar+gzip", - Digest: fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))), - }}, + testManifest := v1.Manifest{ + Layers: []specs.Descriptor{*layer}, } - err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) + + err = unpackManifest(ctx, &testManifest, engine, filepath.Join(tmp1, "rootfs")) if err != nil { t.Fatal(err) } @@ -124,56 +113,38 @@ func TestUnpackLayer(t *testing.T) { } func TestUnpackLayerRemovePartialyUnpackedFile(t *testing.T) { + ctx := context.Background() + // generate a tar file has duplicate entry which will failed on unpacking tmp1, err := ioutil.TempDir("", "test-layer") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp1) - err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700) - if err != nil { - t.Fatal(err) - } - tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar") - f, err := os.Create(tarfile) - if err != nil { - t.Fatal(err) - } - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test"))) - tw.WriteHeader(&tar.Header{Name: "test", Size: 5, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test1"))) - tw.Close() - gw.Close() - f.Close() - // generate sha256 hash - h := sha256.New() - file, err := os.Open(tarfile) + err = imagelayout.CreateDir(ctx, tmp1) if err != nil { t.Fatal(err) } - defer file.Close() - _, err = io.Copy(h, file) + + engine, err := caslayout.NewEngine(ctx, tmp1) if err != nil { t.Fatal(err) } - err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil)))) + defer engine.Close() + + layer, err := createTarBlob(ctx, engine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + tarContent{&tar.Header{Name: "test", Size: 5, Mode: 0600}, []byte("test1")}, + }) if err != nil { t.Fatal(err) } - testManifest := manifest{ - Layers: []descriptor{descriptor{ - MediaType: "application/vnd.oci.image.layer.tar+gzip", - Digest: fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))), - }}, + testManifest := v1.Manifest{ + Layers: []specs.Descriptor{*layer}, } - err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) + err = unpackManifest(ctx, &testManifest, engine, filepath.Join(tmp1, "rootfs")) if err != nil && !strings.Contains(err.Error(), "duplicate entry for") { t.Fatal(err) } diff --git a/image/reader.go b/image/reader.go deleted file mode 100644 index 078db5a..0000000 --- a/image/reader.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" -) - -type reader interface { - Get(desc descriptor) (io.ReadCloser, error) -} - -type tarReader struct { - name string -} - -func (r *tarReader) Get(desc descriptor) (io.ReadCloser, error) { - f, err := os.Open(r.name) - if err != nil { - return nil, err - } - defer f.Close() - - tr := tar.NewReader(f) -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return nil, err - } - if hdr.Name == filepath.Join("blobs", desc.algo(), desc.hash()) && - !hdr.FileInfo().IsDir() { - buf, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - return ioutil.NopCloser(bytes.NewReader(buf)), nil - } - } - - return nil, fmt.Errorf("object not found") -} - -type layoutReader struct { - root string -} - -func (r *layoutReader) Get(desc descriptor) (io.ReadCloser, error) { - name := filepath.Join(r.root, "blobs", desc.algo(), desc.hash()) - - info, err := os.Stat(name) - if err != nil { - return nil, err - } - - if info.IsDir() { - return nil, fmt.Errorf("object is dir") - } - - return os.Open(name) -} diff --git a/image/refs/interface.go b/image/refs/interface.go new file mode 100644 index 0000000..59d5b2c --- /dev/null +++ b/image/refs/interface.go @@ -0,0 +1,83 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package refs implements generic name-based reference access. +package refs + +import ( + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// ListNameCallback templates an Engine.List callback used for +// processing names. See Engine.List for more details. +type ListNameCallback func(ctx context.Context, name string) (err error) + +// Engine represents a name-based reference storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. +type Engine interface { + + // Put adds a new reference to the store. The action is idempotent; + // a nil return means "that descriptor is stored at NAME" without + // implying "because of your Put()". + Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) + + // Get returns a reference from the store. Returns os.ErrNotExist + // if the name is not found. + Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) + + // List returns available names from the store. + // + // Results are sorted alphabetically. + // + // Arguments: + // + // * ctx: gives callers a way to gracefully cancel a long-running + // list. + // * prefix: limits the result set to names starting with that + // value. + // * size: limits the length of the result set to the first 'size' + // matches. A value of -1 means "all results". + // * from: shifts the result set to start from the 'from'th match. + // * nameCallback: called for every matching name. List returns any + // errors returned by nameCallback and aborts further listing. + // + // For example, a store with names like: + // + // * 123 + // * abcd + // * abce + // * abcf + // * abcg + // + // will have the following call/result pairs: + // + // * List(ctx, "", -1, 0, printName) -> "123", "abcd", "abce", "abcf", "abcg" + // * List(ctx, "", 2, 0, printName) -> "123", "abcd" + // * List(ctx, "", 2, 1, printName) -> "abcd", "abce" + // * List(ctx,"abc", 2, 1, printName) -> "abce", "abcf" + List(ctx context.Context, prefix string, size int, from int, nameCallback ListNameCallback) (err error) + + // Delete removes a reference from the store. The action is + // idempotent; a nil return means "that reference is not in the + // store" without implying "because of your Delete()". + Delete(ctx context.Context, name string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/refs/layout/dir.go b/image/refs/layout/dir.go new file mode 100644 index 0000000..af33fe4 --- /dev/null +++ b/image/refs/layout/dir.go @@ -0,0 +1,167 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/layout" + "github.com/opencontainers/image-tools/image/refs" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// DirEngine is a refs.Engine backed by a directory. +type DirEngine struct { + path string + temp string +} + +// NewDirEngine returns a new DirEngine. +func NewDirEngine(ctx context.Context, path string) (eng refs.Engine, err error) { + engine := &DirEngine{ + path: path, + } + + err = layout.CheckDirVersion(ctx, engine.path) + if err != nil { + return nil, err + } + + tempDir, err := ioutil.TempDir(path, "tmp-") + if err != nil { + return nil, err + } + engine.temp = tempDir + + return engine, nil +} + +// Put adds a new reference to the store. +func (engine *DirEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { + var file *os.File + file, err = ioutil.TempFile(engine.temp, "ref-") + if err != nil { + return err + } + defer func() { + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + err = errors.Wrap(err, err2.Error()) + } + } + }() + defer file.Close() + + encoder := json.NewEncoder(file) + err = encoder.Encode(descriptor) + if err != nil { + return err + } + + err = file.Close() + if err != nil { + return err + } + + targetName, err := refPath(name, string(os.PathSeparator)) + if err != nil { + return err + } + + path := filepath.Join(engine.path, targetName) + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return err + } + + return os.Rename(file.Name(), path) +} + +// Get returns a reference from the store. +func (engine *DirEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { + targetName, err := refPath(name, string(os.PathSeparator)) + if err != nil { + return nil, err + } + + var file *os.File + file, err = os.Open(filepath.Join(engine.path, targetName)) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(file) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err + } + return &desc, nil +} + +// List returns available names from the store. +func (engine *DirEngine) List(ctx context.Context, prefix string, size int, from int, nameCallback refs.ListNameCallback) (err error) { + var i = 0 + + pathPrefix, err := refPath(prefix, string(os.PathSeparator)) + if err != nil { + return nil + } + var files []os.FileInfo + files, err = ioutil.ReadDir(filepath.Join(engine.path, filepath.Dir(pathPrefix))) + if err != nil { + return err + } + for _, file := range files { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + name := file.Name() + if strings.HasPrefix(name, prefix) { + i++ + if i > from { + err = nameCallback(ctx, name) + if err != nil { + return err + } + if i-from == size { + return nil + } + } + } + } + + return nil +} + +// Delete removes a reference from the store. +func (engine *DirEngine) Delete(ctx context.Context, name string) (err error) { + return layout.DirDelete(ctx, engine.path, name, refPath) +} + +// Close releases resources held by the engine. +func (engine *DirEngine) Close() (err error) { + return os.RemoveAll(engine.temp) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go new file mode 100644 index 0000000..41c2fb5 --- /dev/null +++ b/image/refs/layout/main.go @@ -0,0 +1,51 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the refs interface using the image-spec's +// image-layout [1]. +// +// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "fmt" + "os" + "strings" + + "github.com/opencontainers/image-tools/image/refs" + "golang.org/x/net/context" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { + engine, err = NewDirEngine(ctx, path) + if err == nil { + return engine, err + } + + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err == nil { + return NewTarEngine(ctx, file) + } + + return nil, fmt.Errorf("unrecognized engine at %q", path) +} + +// refPath returns the PATH to the NAME reference. SEPARATOR selects +// the path separator used between components. +func refPath(name string, separator string) (path string, err error) { + components := []string{".", "refs", name} + return strings.Join(components, separator), nil +} diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go new file mode 100644 index 0000000..d23cd0e --- /dev/null +++ b/image/refs/layout/tar.go @@ -0,0 +1,143 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "io" + "os" + "strings" + + "github.com/opencontainers/image-spec/specs-go" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + "github.com/opencontainers/image-tools/image/refs" + "golang.org/x/net/context" +) + +// TarEngine is a refs.Engine backed by a tar file. +type TarEngine struct { + file caslayout.ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(ctx context.Context, file caslayout.ReadWriteSeekCloser) (eng refs.Engine, err error) { + engine := &TarEngine{ + file: file, + } + + err = imagelayout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + + return engine, nil +} + +// Put adds a new reference to the store. +func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { + data, err := json.Marshal(descriptor) + if err != nil { + return err + } + + size := int64(len(data)) + reader := bytes.NewReader(data) + targetName, err := refPath(name, "/") + if err != nil { + return err + } + return imagelayout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) +} + +// Get returns a reference from the store. +func (engine *TarEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { + targetName, err := refPath(name, "/") + if err != nil { + return nil, err + } + + _, tarReader, err := imagelayout.TarEntryByName(ctx, engine.file, targetName) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(tarReader) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err + } + return &desc, nil +} + +// List returns available names from the store. +func (engine *TarEngine) List(ctx context.Context, prefix string, size int, from int, nameCallback refs.ListNameCallback) (err error) { + var i = 0 + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil + } + + pathPrefix, err := refPath(prefix, "/") + if err != nil { + return nil + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + return nil + } else if err != nil { + return err + } + + if strings.HasPrefix(header.Name, pathPrefix) && len(header.Name) > 7 { + i++ + if i > from { + err = nameCallback(ctx, header.Name[7:]) + if err != nil { + return err + } + if i-from == size { + return nil + } + } + } + } +} + +// Delete removes a reference from the store. +func (engine *TarEngine) Delete(ctx context.Context, name string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} diff --git a/image/walker.go b/image/walker.go deleted file mode 100644 index 27acfac..0000000 --- a/image/walker.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -var ( - errEOW = fmt.Errorf("end of walk") // error to signal stop walking -) - -// walkFunc is a function type that gets called for each file or directory visited by the Walker. -type walkFunc func(path string, _ os.FileInfo, _ io.Reader) error - -// walker is the interface that walks through a file tree, -// calling walk for each file or directory in the tree. -type walker interface { - walk(walkFunc) error - reader -} - -type tarWalker struct { - r io.ReadSeeker - tarReader -} - -// newTarWalker returns a Walker that walks through .tar files. -func newTarWalker(tarFile string, r io.ReadSeeker) walker { - return &tarWalker{r, tarReader{name: tarFile}} -} - -func (w *tarWalker) walk(f walkFunc) error { - if _, err := w.r.Seek(0, os.SEEK_SET); err != nil { - return errors.Wrapf(err, "unable to reset") - } - - tr := tar.NewReader(w.r) - -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return errors.Wrapf(err, "error advancing tar stream") - } - - info := hdr.FileInfo() - if err := f(hdr.Name, info, tr); err != nil { - return err - } - } - - return nil -} - -type eofReader struct{} - -func (eofReader) Read(_ []byte) (int, error) { - return 0, io.EOF -} - -type pathWalker struct { - root string - layoutReader -} - -// newPathWalker returns a Walker that walks through directories -// starting at the given root path. It does not follow symlinks. -func newPathWalker(root string) walker { - return &pathWalker{root, layoutReader{root: root}} -} - -func (w *pathWalker) walk(f walkFunc) error { - return filepath.Walk(w.root, func(path string, info os.FileInfo, err error) error { - // MUST check error value, to make sure the `os.FileInfo` is available. - // Otherwise panic risk will exist. - if err != nil { - return errors.Wrap(err, "error walking path") - } - - rel, err := filepath.Rel(w.root, path) - if err != nil { - return errors.Wrap(err, "error walking path") // err from filepath.Walk includes path name - } - - if info.IsDir() { // behave like a tar reader for directories - return f(rel, info, eofReader{}) - } - - file, err := os.Open(path) - if err != nil { - return errors.Wrap(err, "unable to open file") // os.Open includes the path - } - defer file.Close() - - return f(rel, info, file) - }) -} diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/golang.org/x/net/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/golang.org/x/net/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/net/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go new file mode 100644 index 0000000..134654c --- /dev/null +++ b/vendor/golang.org/x/net/context/context.go @@ -0,0 +1,156 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package context defines the Context type, which carries deadlines, +// cancelation signals, and other request-scoped values across API boundaries +// and between processes. +// +// Incoming requests to a server should create a Context, and outgoing calls to +// servers should accept a Context. The chain of function calls between must +// propagate the Context, optionally replacing it with a modified copy created +// using WithDeadline, WithTimeout, WithCancel, or WithValue. +// +// Programs that use Contexts should follow these rules to keep interfaces +// consistent across packages and enable static analysis tools to check context +// propagation: +// +// Do not store Contexts inside a struct type; instead, pass a Context +// explicitly to each function that needs it. The Context should be the first +// parameter, typically named ctx: +// +// func DoSomething(ctx context.Context, arg Arg) error { +// // ... use ctx ... +// } +// +// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// if you are unsure about which Context to use. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +// +// The same Context may be passed to functions running in different goroutines; +// Contexts are safe for simultaneous use by multiple goroutines. +// +// See http://blog.golang.org/context for example code for a server that uses +// Contexts. +package context // import "golang.org/x/net/context" + +import "time" + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out chan<- Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// Background returns a non-nil, empty Context. It is never canceled, has no +// values, and has no deadline. It is typically used by the main function, +// initialization, and tests, and as the top-level Context for incoming +// requests. +func Background() Context { + return background +} + +// TODO returns a non-nil, empty Context. Code should use context.TODO when +// it's unclear which Context to use or it is not yet available (because the +// surrounding function has not yet been extended to accept a Context +// parameter). TODO is recognized by static analysis tools that determine +// whether Contexts are propagated correctly in a program. +func TODO() Context { + return todo +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go new file mode 100644 index 0000000..f8cda19 --- /dev/null +++ b/vendor/golang.org/x/net/context/go17.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package context + +import ( + "context" // standard library's context, as of Go 1.7 + "time" +) + +var ( + todo = context.TODO() + background = context.Background() +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = context.Canceled + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = context.DeadlineExceeded + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + ctx, f := context.WithCancel(parent) + return ctx, CancelFunc(f) +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + ctx, f := context.WithDeadline(parent, deadline) + return ctx, CancelFunc(f) +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return context.WithValue(parent, key, val) +} diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go new file mode 100644 index 0000000..5a30aca --- /dev/null +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -0,0 +1,300 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.7 + +package context + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// struct{}, since vars of this type must have distinct addresses. +type emptyCtx int + +func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +func (*emptyCtx) Done() <-chan struct{} { + return nil +} + +func (*emptyCtx) Err() error { + return nil +} + +func (*emptyCtx) Value(key interface{}) interface{} { + return nil +} + +func (e *emptyCtx) String() string { + switch e { + case background: + return "context.Background" + case todo: + return "context.TODO" + } + return "unknown empty Context" +} + +var ( + background = new(emptyCtx) + todo = new(emptyCtx) +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = errors.New("context canceled") + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = errors.New("context deadline exceeded") + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + c := newCancelCtx(parent) + propagateCancel(parent, c) + return c, func() { c.cancel(true, Canceled) } +} + +// newCancelCtx returns an initialized cancelCtx. +func newCancelCtx(parent Context) *cancelCtx { + return &cancelCtx{ + Context: parent, + done: make(chan struct{}), + } +} + +// propagateCancel arranges for child to be canceled when parent is. +func propagateCancel(parent Context, child canceler) { + if parent.Done() == nil { + return // parent is never canceled + } + if p, ok := parentCancelCtx(parent); ok { + p.mu.Lock() + if p.err != nil { + // parent has already been canceled + child.cancel(false, p.err) + } else { + if p.children == nil { + p.children = make(map[canceler]bool) + } + p.children[child] = true + } + p.mu.Unlock() + } else { + go func() { + select { + case <-parent.Done(): + child.cancel(false, parent.Err()) + case <-child.Done(): + } + }() + } +} + +// parentCancelCtx follows a chain of parent references until it finds a +// *cancelCtx. This function understands how each of the concrete types in this +// package represents its parent. +func parentCancelCtx(parent Context) (*cancelCtx, bool) { + for { + switch c := parent.(type) { + case *cancelCtx: + return c, true + case *timerCtx: + return c.cancelCtx, true + case *valueCtx: + parent = c.Context + default: + return nil, false + } + } +} + +// removeChild removes a context from its parent. +func removeChild(parent Context, child canceler) { + p, ok := parentCancelCtx(parent) + if !ok { + return + } + p.mu.Lock() + if p.children != nil { + delete(p.children, child) + } + p.mu.Unlock() +} + +// A canceler is a context type that can be canceled directly. The +// implementations are *cancelCtx and *timerCtx. +type canceler interface { + cancel(removeFromParent bool, err error) + Done() <-chan struct{} +} + +// A cancelCtx can be canceled. When canceled, it also cancels any children +// that implement canceler. +type cancelCtx struct { + Context + + done chan struct{} // closed by the first cancel call. + + mu sync.Mutex + children map[canceler]bool // set to nil by the first cancel call + err error // set to non-nil by the first cancel call +} + +func (c *cancelCtx) Done() <-chan struct{} { + return c.done +} + +func (c *cancelCtx) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *cancelCtx) String() string { + return fmt.Sprintf("%v.WithCancel", c.Context) +} + +// cancel closes c.done, cancels each of c's children, and, if +// removeFromParent is true, removes c from its parent's children. +func (c *cancelCtx) cancel(removeFromParent bool, err error) { + if err == nil { + panic("context: internal error: missing cancel error") + } + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return // already canceled + } + c.err = err + close(c.done) + for child := range c.children { + // NOTE: acquiring the child's lock while holding parent's lock. + child.cancel(false, err) + } + c.children = nil + c.mu.Unlock() + + if removeFromParent { + removeChild(c.Context, c) + } +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { + // The current deadline is already sooner than the new one. + return WithCancel(parent) + } + c := &timerCtx{ + cancelCtx: newCancelCtx(parent), + deadline: deadline, + } + propagateCancel(parent, c) + d := deadline.Sub(time.Now()) + if d <= 0 { + c.cancel(true, DeadlineExceeded) // deadline has already passed + return c, func() { c.cancel(true, Canceled) } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.timer = time.AfterFunc(d, func() { + c.cancel(true, DeadlineExceeded) + }) + } + return c, func() { c.cancel(true, Canceled) } +} + +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then +// delegating to cancelCtx.cancel. +type timerCtx struct { + *cancelCtx + timer *time.Timer // Under cancelCtx.mu. + + deadline time.Time +} + +func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { + return c.deadline, true +} + +func (c *timerCtx) String() string { + return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) +} + +func (c *timerCtx) cancel(removeFromParent bool, err error) { + c.cancelCtx.cancel(false, err) + if removeFromParent { + // Remove this timerCtx from its parent cancelCtx's children. + removeChild(c.cancelCtx.Context, c) + } + c.mu.Lock() + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.mu.Unlock() +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return &valueCtx{parent, key, val} +} + +// A valueCtx carries a key-value pair. It implements Value for that key and +// delegates all other calls to the embedded Context. +type valueCtx struct { + Context + key, val interface{} +} + +func (c *valueCtx) String() string { + return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) +} + +func (c *valueCtx) Value(key interface{}) interface{} { + if c.key == key { + return c.val + } + return c.Context.Value(key) +}