diff --git a/schema/config_test.go b/schema/config_test.go index 69ae067c0..467cf64c3 100644 --- a/schema/config_test.go +++ b/schema/config_test.go @@ -18,7 +18,9 @@ import ( "strings" "testing" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go/v1" ) func TestConfig(t *testing.T) { @@ -210,9 +212,14 @@ func TestConfig(t *testing.T) { fail: false, }, } { - r := strings.NewReader(tt.config) - err := schema.MediaTypeImageConfig.Validate(r) - + configBytes := []byte(tt.config) + reader := strings.NewReader(tt.config) + descriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: digest.FromBytes(configBytes).String(), + Size: int64(len(configBytes)), + } + err := schema.Validate(reader, &descriptor, true) if got := err != nil; tt.fail != got { t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err) } diff --git a/schema/manifest.go b/schema/manifest.go new file mode 100644 index 000000000..9da305345 --- /dev/null +++ b/schema/manifest.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// ValidateManifest validates the given CAS blob as +// application/vnd.oci.image.manifest.v1+json. Calls +// ValidateJSONSchema as well. +func ValidateManifest(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) { + if descriptor.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("unexpected descriptor media type: %q", descriptor.MediaType) + } + + buffer, err := ioutil.ReadAll(blob) + if err != nil { + return errors.Wrapf(err, "unable to read %s", descriptor.Digest) + } + + err = ValidateJSONSchema(bytes.NewReader(buffer), descriptor, strict) + if err != nil { + return err + } + + header := v1.Manifest{} + err = json.Unmarshal(buffer, &header) + if err != nil { + return errors.Wrap(err, "manifest format mismatch") + } + + if header.Config.MediaType != v1.MediaTypeImageConfig { + error := fmt.Errorf("warning: config %s has an unknown media type: %s\n", header.Config.Digest, header.Config.MediaType) + if strict { + return error + } + fmt.Println(error) + } + + for _, layer := range header.Layers { + if layer.MediaType != v1.MediaTypeImageLayer && + layer.MediaType != v1.MediaTypeImageLayerNonDistributable { + error := fmt.Errorf("warning: layer %s has an unknown media type: %s\n", layer.Digest, layer.MediaType) + if strict { + return error + } + fmt.Println(error) + } + } + + return nil +} diff --git a/schema/manifest_backwards_compatibility_test.go b/schema/manifest_backwards_compatibility_test.go index ad94fd3a8..94ed72634 100644 --- a/schema/manifest_backwards_compatibility_test.go +++ b/schema/manifest_backwards_compatibility_test.go @@ -15,12 +15,11 @@ package schema_test import ( - "crypto/sha256" - "encoding/hex" - "fmt" + "bytes" "strings" "testing" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -45,13 +44,13 @@ func convertFormats(input string) string { func TestBackwardsCompatibilityManifestList(t *testing.T) { for i, tt := range []struct { - manifest string - digest string - fail bool + manifestList string + digest string + fail bool }{ { digest: "sha256:219f4b61132fe9d09b0ec5c15517be2ca712e4744b0e0cc3be71295b35b2a467", - manifest: `{ + manifestList: `{ "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", "manifests": [ @@ -110,16 +109,18 @@ func TestBackwardsCompatibilityManifestList(t *testing.T) { fail: false, }, } { - sum := sha256.Sum256([]byte(tt.manifest)) - got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:])) - if tt.digest != got { - t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got) + err := schema.ValidateByteDigest([]byte(tt.manifestList), &v1.Descriptor{Digest: tt.digest}) + if err != nil { + t.Fatal(err) } - - manifest := convertFormats(tt.manifest) - r := strings.NewReader(manifest) - err := schema.MediaTypeManifestList.Validate(r) - + manifestList := []byte(convertFormats(tt.manifestList)) + reader := bytes.NewReader(manifestList) + descriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageManifestList, + Digest: digest.FromBytes(manifestList).String(), + Size: int64(len(manifestList)), + } + err = schema.Validate(reader, &descriptor, true) if got := err != nil; tt.fail != got { t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err) } @@ -130,6 +131,7 @@ func TestBackwardsCompatibilityManifest(t *testing.T) { for i, tt := range []struct { manifest string digest string + strict bool fail bool }{ // manifest pulled from docker hub using hash value @@ -170,19 +172,22 @@ func TestBackwardsCompatibilityManifest(t *testing.T) { } ] }`, - fail: false, + strict: false, // unrecognized config media type application/octet-stream + fail: false, }, } { - sum := sha256.Sum256([]byte(tt.manifest)) - got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:])) - if tt.digest != got { - t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got) + err := schema.ValidateByteDigest([]byte(tt.manifest), &v1.Descriptor{Digest: tt.digest}) + if err != nil { + t.Fatal(err) } - - manifest := convertFormats(tt.manifest) - r := strings.NewReader(manifest) - err := schema.MediaTypeManifest.Validate(r) - + manifest := []byte(convertFormats(tt.manifest)) + reader := bytes.NewReader(manifest) + descriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageManifest, + Digest: digest.FromBytes(manifest).String(), + Size: int64(len(manifest)), + } + err = schema.Validate(reader, &descriptor, tt.strict) if got := err != nil; tt.fail != got { t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err) } @@ -213,16 +218,18 @@ func TestBackwardsCompatibilityConfig(t *testing.T) { fail: false, }, } { - sum := sha256.Sum256([]byte(tt.config)) - got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:])) - if tt.digest != got { - t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got) + err := schema.ValidateByteDigest([]byte(tt.config), &v1.Descriptor{Digest: tt.digest}) + if err != nil { + t.Fatal(err) } - - config := convertFormats(tt.config) - r := strings.NewReader(config) - err := schema.MediaTypeImageConfig.Validate(r) - + config := []byte(convertFormats(tt.config)) + reader := bytes.NewReader(config) + descriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: digest.FromBytes(config).String(), + Size: int64(len(config)), + } + err = schema.Validate(reader, &descriptor, true) if got := err != nil; tt.fail != got { t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err) } diff --git a/schema/manifest_test.go b/schema/manifest_test.go index 02b4ad34b..371717441 100644 --- a/schema/manifest_test.go +++ b/schema/manifest_test.go @@ -18,12 +18,15 @@ import ( "strings" "testing" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go/v1" ) func TestManifest(t *testing.T) { for i, tt := range []struct { manifest string + strict bool fail bool }{ // expected failure: mediaType does not match pattern @@ -46,7 +49,8 @@ func TestManifest(t *testing.T) { ] } `, - fail: true, + strict: true, + fail: true, }, // expected failure: config.size is a string, expected integer @@ -69,7 +73,8 @@ func TestManifest(t *testing.T) { ] } `, - fail: true, + strict: true, + fail: true, }, // expected failure: layers.size is string, expected integer @@ -92,7 +97,56 @@ func TestManifest(t *testing.T) { ] } `, - fail: true, + strict: true, + fail: true, + }, + + // expected failure: unrecognized layer media type and strict is true + { + manifest: ` +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 1470, + "digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b" + }, + "layers": [ + { + "mediaType": "application/vnd.other.layer", + "size": "675598", + "digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b" + } + ] +} +`, + strict: true, + fail: true, + }, + + // expected success: unrecognized layer media type, but strict is false + { + manifest: ` +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 1470, + "digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b" + }, + "layers": [ + { + "mediaType": "application/vnd.other.layer", + "size": "675598", + "digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b" + } + ] +} +`, + strict: false, + fail: true, }, // valid manifest with optional fields @@ -129,7 +183,8 @@ func TestManifest(t *testing.T) { } } `, - fail: false, + strict: true, + fail: false, }, // valid manifest with only required fields @@ -182,9 +237,14 @@ func TestManifest(t *testing.T) { fail: true, }, } { - r := strings.NewReader(tt.manifest) - err := schema.MediaTypeManifest.Validate(r) - + manifestBytes := []byte(tt.manifest) + reader := strings.NewReader(tt.manifest) + descriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageManifest, + Digest: digest.FromBytes(manifestBytes).String(), + Size: int64(len(manifestBytes)), + } + err := schema.Validate(reader, &descriptor, tt.strict) if got := err != nil; tt.fail != got { t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err) } diff --git a/schema/schema.go b/schema/schema.go index 1ca6312c4..d85a41a95 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -20,26 +20,17 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) -// Media types for the OCI image formats -const ( - MediaTypeDescriptor Validator = v1.MediaTypeDescriptor - MediaTypeManifest Validator = v1.MediaTypeImageManifest - MediaTypeManifestList Validator = v1.MediaTypeImageManifestList - MediaTypeImageConfig Validator = v1.MediaTypeImageConfig - MediaTypeImageLayer unimplemented = v1.MediaTypeImageLayer -) - var ( // fs stores the embedded http.FileSystem // having the OCI JSON schema files in root "/". fs = _escFS(false) - // specs maps OCI schema media types to schema files. - specs = map[Validator]string{ - MediaTypeDescriptor: "content-descriptor.json", - MediaTypeManifest: "image-manifest-schema.json", - MediaTypeManifestList: "manifest-list-schema.json", - MediaTypeImageConfig: "config-schema.json", + // Schemas maps OCI media types to JSON Schema files. + Schemas = map[string]string{ + v1.MediaTypeDescriptor: "content-descriptor.json", + v1.MediaTypeImageManifest: "image-manifest-schema.json", + v1.MediaTypeImageManifestList: "manifest-list-schema.json", + v1.MediaTypeImageConfig: "config-schema.json", } ) diff --git a/schema/spec_test.go b/schema/spec_test.go index 2a6f4a493..c8ff84185 100644 --- a/schema/spec_test.go +++ b/schema/spec_test.go @@ -24,7 +24,9 @@ import ( "strings" "testing" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/russross/blackfriday" ) @@ -73,7 +75,15 @@ func validate(t *testing.T, name string) { continue } - err = schema.Validator(example.Mediatype).Validate(strings.NewReader(example.Body)) + bodyBytes := []byte(example.Body) + bodyDigest := digest.FromBytes(bodyBytes).String() + reader := strings.NewReader(example.Body) + descriptor := v1.Descriptor{ + MediaType: example.Mediatype, + Digest: bodyDigest, + Size: int64(len(bodyBytes)), + } + err = schema.Validate(reader, &descriptor, true) if err == nil { printFields(t, "ok", example.Mediatype, example.Title) t.Log(example.Body, "---") diff --git a/schema/validator.go b/schema/validator.go index 432e7b992..1e9ab3d09 100644 --- a/schema/validator.go +++ b/schema/validator.go @@ -16,60 +16,81 @@ package schema import ( "bytes" - "encoding/json" "fmt" "io" "io/ioutil" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/xeipuuv/gojsonschema" ) -// Validator wraps a media type string identifier -// and implements validation against a JSON schema. -type Validator string - -type validateDescendantsFunc func(r io.Reader) error - -var mapValidateDescendants = map[Validator]validateDescendantsFunc{ - MediaTypeManifest: validateManifestDescendants, -} - -// ValidationError contains all the errors that happened during validation. +// ValidationError contains all the errors that happened during +// validation. type ValidationError struct { Errs []error } +// Validator is a template for validating a CAS blob. The 'strict' +// parameter distinguishes between compliant blobs (which should only +// pass when strict is false) and blobs that only use features which +// the spec requires implementations to support (which should pass +// regardless of strict). +type Validator func(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) + +// Validators is a map from media types to an appropriate Validator +// function. +var Validators = map[string]Validator{ + v1.MediaTypeDescriptor: ValidateJSONSchema, + v1.MediaTypeImageManifestList: ValidateJSONSchema, + v1.MediaTypeImageManifest: ValidateManifest, + v1.MediaTypeImageConfig: ValidateJSONSchema, +} + func (e ValidationError) Error() string { return fmt.Sprintf("%v", e.Errs) } -// Validate validates the given reader against the schema of the wrapped media type. -func (v Validator) Validate(src io.Reader) error { - buf, err := ioutil.ReadAll(src) +// Validate retrieves the appropriate Validator from Validators and +// uses it to validate the given CAS blob. Validate uses the +// Validator template; see the Validator docs for usage information. +func Validate(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) { + validator, ok := Validators[descriptor.MediaType] + if !ok { + return fmt.Errorf("unrecognized media type %q", descriptor.MediaType) + } + return validator(blob, descriptor, strict) +} + +// ValidateJSONSchema validates the given CAS blob against the schema +// for the descriptor's media type. Calls ValidateByteSize and +// ValidateByteDigest as well. +func ValidateJSONSchema(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) { + buffer, err := ioutil.ReadAll(blob) + if err != nil { + return errors.Wrapf(err, "unable to read %s", descriptor.Digest) + } + + err = ValidateByteSize(buffer, descriptor) if err != nil { - return errors.Wrap(err, "unable to read the document file") + return err } - if f, ok := mapValidateDescendants[v]; ok { - if f == nil { - return fmt.Errorf("internal error: mapValidateDescendents[%q] is nil", v) - } - err = f(bytes.NewReader(buf)) - if err != nil { - return err - } + err = ValidateByteDigest(buffer, descriptor) + if err != nil { + return err } - sl := gojsonschema.NewReferenceLoaderFileSystem("file:///"+specs[v], fs) - ml := gojsonschema.NewStringLoader(string(buf)) + url := "file:///" + Schemas[descriptor.MediaType] + schemaLoader := gojsonschema.NewReferenceLoaderFileSystem(url, fs) + docLoader := gojsonschema.NewStringLoader(string(buffer)) - result, err := gojsonschema.Validate(sl, ml) + result, err := gojsonschema.Validate(schemaLoader, docLoader) if err != nil { return errors.Wrapf( - WrapSyntaxError(bytes.NewReader(buf), err), - "schema %s: unable to validate", v) + WrapSyntaxError(bytes.NewReader(buffer), err), + "unable to validate JSON Schema for %s", descriptor.Digest) } if result.Valid() { @@ -77,8 +98,8 @@ func (v Validator) Validate(src io.Reader) error { } errs := make([]error, 0, len(result.Errors())) - for _, desc := range result.Errors() { - errs = append(errs, fmt.Errorf("%s", desc)) + for _, description := range result.Errors() { + errs = append(errs, fmt.Errorf("%s", description)) } return ValidationError{ @@ -86,34 +107,32 @@ func (v Validator) Validate(src io.Reader) error { } } -type unimplemented string - -func (v unimplemented) Validate(src io.Reader) error { - return fmt.Errorf("%s: unimplemented", v) -} - -func validateManifestDescendants(r io.Reader) error { - header := v1.Manifest{} - - buf, err := ioutil.ReadAll(r) +// ValidateByteDigest checks the digest of blob against the expected +// descriptor.Digest. +func ValidateByteDigest(blob []byte, descriptor *v1.Descriptor) (err error) { + parsed, err := digest.Parse(descriptor.Digest) if err != nil { - return errors.Wrapf(err, "error reading the io stream") + return err } - - err = json.Unmarshal(buf, &header) - if err != nil { - return errors.Wrap(err, "manifest format mismatch") + algorithm := parsed.Algorithm() + if !algorithm.Available() { + return fmt.Errorf("unsupported digest algorithm for %s", descriptor.Digest) } - - if header.Config.MediaType != string(v1.MediaTypeImageConfig) { - fmt.Printf("warning: config %s has an unknown media type: %s\n", header.Config.Digest, header.Config.MediaType) + actualDigest := algorithm.FromBytes(blob).String() + if actualDigest != descriptor.Digest { + return fmt.Errorf("unexpected digest for %s: %s", descriptor.Digest, actualDigest) } - for _, layer := range header.Layers { - if layer.MediaType != string(v1.MediaTypeImageLayer) && - layer.MediaType != string(v1.MediaTypeImageLayerNonDistributable) { - fmt.Printf("warning: layer %s has an unknown media type: %s\n", layer.Digest, layer.MediaType) - } + return nil +} + +// ValidateByteSize checks the size of blob against the expected +// descriptor.Size. This isn't very complicated; the function is +// mostly useful for generating consistent error messages. +func ValidateByteSize(blob []byte, descriptor *v1.Descriptor) (err error) { + if descriptor.Size > 0 && int64(len(blob)) != descriptor.Size { + return fmt.Errorf("unexpected size for %s: %d != %d", descriptor.Digest, len(blob), descriptor.Size) } + return nil }