From dab7bc68ac86c343cae74abd561db9fb9c1ba14b Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 14 Oct 2022 15:47:18 +0800 Subject: [PATCH 01/17] draft push artifact Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 116 +++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 8d0c3dcf..1f41390b 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -29,7 +29,9 @@ import ( "sync/atomic" "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/internal/cas" "oras.land/oras-go/v2/internal/httputil" @@ -877,8 +879,45 @@ func (s *manifestStore) Fetch(ctx context.Context, target ocispec.Descriptor) (r } // Push pushes the content, matching the expected descriptor. -func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { - return s.push(ctx, expected, content, expected.Digest.String()) +func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, body io.Reader) error { + var buf bytes.Buffer + lr := io.LimitReader(body, s.repo.MaxMetadataBytes) + tr := io.TeeReader(lr, &buf) + isArtifact, artifactDesc, err := checkArtifact(expected, tr) + if err != nil { + return err + } + + if err := s.push(ctx, expected, &buf, expected.Digest.String()); err != nil { + return err + } + if isArtifact { + return s.indexReferrers(ctx, artifactDesc) + } + return nil +} + +func checkArtifact(expected ocispec.Descriptor, r io.Reader) (bool, ocispec.Descriptor, error) { + manifestJSON, err := content.ReadAll(r, expected) + if err != nil { + return false, ocispec.Descriptor{}, err + } + type manifest struct { + ArtifactType string `json:"artifactType"` + Subject *ocispec.Descriptor `json:"subject,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + } + var m manifest + if err := json.Unmarshal(manifestJSON, &m); err != nil { + return false, ocispec.Descriptor{}, err + } + + if m.Subject != nil { + expected.ArtifactType = m.ArtifactType + expected.Annotations = m.Annotations + return true, expected, nil + } + return false, expected, nil } // Exists returns true if the described content exists. @@ -992,6 +1031,7 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc if err != nil { return err } + // TODO: clone body return s.push(ctx, expected, content, ref.Reference) } @@ -1051,6 +1091,78 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } +func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor) error { + ok, err := s.repo.testReferrersAPI(ctx, desc) + if err != nil { + return err + } + if ok { + // no need to index + return nil + } + + var existingReferrers []ocispec.Descriptor + // TODO: call referrersTagSchema + if err != nil && !errors.Is(err, errdef.ErrNotFound) { + // If that pull returns a manifest other than the expected image index, the client SHOULD report a failure and skip the remaining steps. + return err + } + + var referrers []ocispec.Descriptor + // If the tag returns a 404, the client MUST begin with an empty image index. + for _, r := range existingReferrers { + if desc.Digest == r.Digest { + // note: desc should already contain the artifact type and annotations + referrers = append(referrers, desc) + } else { + referrers = append(referrers, r) + } + } + + index := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: referrers, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return err // TODO: format + } + referrersTag := "" // TODO: build referrers tag + + // TODO: concurrency? + return s.push(ctx, desc, bytes.NewReader(indexJSON), referrersTag) +} + +func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descriptor) (bool, error) { + ref := r.Reference + ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull) + ref.Reference = desc.Digest.String() + // url := buildReferrersURL(r.PlainHTTP, ref, "") + url := "" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + resp, err := r.client().Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + return false, errutil.ParseErrorResponse(resp) + } +} + // ParseReference parses a reference to a fully qualified reference. func (s *manifestStore) ParseReference(reference string) (registry.Reference, error) { return s.repo.ParseReference(reference) From 4645909aad69a2c9d3e5e33267c1d40aa96cd963 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Thu, 20 Oct 2022 11:22:31 +0800 Subject: [PATCH 02/17] draft indexReferrers Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 131 +++++++++++++++++----------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 1f41390b..ee7c29e3 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -883,24 +883,22 @@ func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, b var buf bytes.Buffer lr := io.LimitReader(body, s.repo.MaxMetadataBytes) tr := io.TeeReader(lr, &buf) - isArtifact, artifactDesc, err := checkArtifact(expected, tr) - if err != nil { + if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { return err } - if err := s.push(ctx, expected, &buf, expected.Digest.String()); err != nil { - return err - } - if isArtifact { - return s.indexReferrers(ctx, artifactDesc) - } - return nil + return s.indexReferrers(ctx, expected, &buf) } -func checkArtifact(expected ocispec.Descriptor, r io.Reader) (bool, ocispec.Descriptor, error) { - manifestJSON, err := content.ReadAll(r, expected) +func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { + if desc.MediaType != ocispec.MediaTypeArtifactManifest && desc.MediaType != ocispec.MediaTypeImageManifest { + // no need to index + return nil + } + + manifestJSON, err := content.ReadAll(r, desc) if err != nil { - return false, ocispec.Descriptor{}, err + return err } type manifest struct { ArtifactType string `json:"artifactType"` @@ -909,15 +907,62 @@ func checkArtifact(expected ocispec.Descriptor, r io.Reader) (bool, ocispec.Desc } var m manifest if err := json.Unmarshal(manifestJSON, &m); err != nil { - return false, ocispec.Descriptor{}, err + return err + } + if m.Subject == nil { + // no need to index + return nil + } + + subject := *m.Subject + // populate ArtifactType and Annotations + desc.ArtifactType = m.ArtifactType + desc.Annotations = m.Annotations + ok, err := s.repo.testReferrersAPI(ctx, subject) + if err != nil { + return err + } + if ok { + // no need to index + return nil } - if m.Subject != nil { - expected.ArtifactType = m.ArtifactType - expected.Annotations = m.Annotations - return true, expected, nil + var existingReferrers []ocispec.Descriptor + // call referrersTagSchema + err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { + existingReferrers = append(existingReferrers, referrers...) + return nil + }) + if err != nil && !errors.Is(err, errdef.ErrNotFound) { + // If that pull returns a manifest other than the expected image index, the client SHOULD report a failure and skip the remaining steps. + return err + } + + var referrers []ocispec.Descriptor + // If the tag returns a 404, the client MUST begin with an empty image index. + for _, r := range existingReferrers { + if r.Digest != desc.Digest { + // TODO: equals? + referrers = append(referrers, r) + } + } + referrers = append(referrers, desc) + + index := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: referrers, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return err // TODO: format } - return false, expected, nil + indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) + referrersTag := buildReferrersTag(subject) + // TODO: concurrency? + return s.push(ctx, indexDesc, bytes.NewReader(indexJSON), referrersTag) } // Exists returns true if the described content exists. @@ -1091,58 +1136,12 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } -func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor) error { - ok, err := s.repo.testReferrersAPI(ctx, desc) - if err != nil { - return err - } - if ok { - // no need to index - return nil - } - - var existingReferrers []ocispec.Descriptor - // TODO: call referrersTagSchema - if err != nil && !errors.Is(err, errdef.ErrNotFound) { - // If that pull returns a manifest other than the expected image index, the client SHOULD report a failure and skip the remaining steps. - return err - } - - var referrers []ocispec.Descriptor - // If the tag returns a 404, the client MUST begin with an empty image index. - for _, r := range existingReferrers { - if desc.Digest == r.Digest { - // note: desc should already contain the artifact type and annotations - referrers = append(referrers, desc) - } else { - referrers = append(referrers, r) - } - } - - index := ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value. does not pertain to OCI or docker version - }, - MediaType: ocispec.MediaTypeImageIndex, - Manifests: referrers, - } - indexJSON, err := json.Marshal(index) - if err != nil { - return err // TODO: format - } - referrersTag := "" // TODO: build referrers tag - - // TODO: concurrency? - return s.push(ctx, desc, bytes.NewReader(indexJSON), referrersTag) -} - func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descriptor) (bool, error) { ref := r.Reference - ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull) ref.Reference = desc.Digest.String() - // url := buildReferrersURL(r.PlainHTTP, ref, "") - url := "" + ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull) + url := buildReferrersURL(r.PlainHTTP, ref, "") req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return false, err From 4331fb199b20f3f103e6f94608c8cdcb856b0a6b Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Thu, 20 Oct 2022 14:59:07 +0800 Subject: [PATCH 03/17] draft refactor Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 76 ++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index ee7c29e3..710cfada 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -26,6 +26,7 @@ import ( "net/http" "strconv" "strings" + "sync" "sync/atomic" "github.com/opencontainers/go-digest" @@ -118,6 +119,23 @@ type Repository struct { // referrersState represents that if the repository supports Referrers API. // default: referrersStateUnknown referrersState referrersState + + referrersTagLocks sync.Map // map[string]sync.Mutex +} + +func (r *Repository) lockReferrersTag(tag string) { + v, _ := r.referrersTagLocks.LoadOrStore(tag, &sync.Mutex{}) + lock := v.(*sync.Mutex) + lock.Lock() +} + +func (r *Repository) unlockReferrersTag(tag string) { + v, ok := r.referrersTagLocks.Load(tag) + if !ok { + return + } + lock := v.(*sync.Mutex) + lock.Unlock() } // NewRepository creates a client to the remote repository identified by a @@ -881,7 +899,7 @@ func (s *manifestStore) Fetch(ctx context.Context, target ocispec.Descriptor) (r // Push pushes the content, matching the expected descriptor. func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, body io.Reader) error { var buf bytes.Buffer - lr := io.LimitReader(body, s.repo.MaxMetadataBytes) + lr := limitReader(body, s.repo.MaxMetadataBytes) tr := io.TeeReader(lr, &buf) if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { return err @@ -891,42 +909,56 @@ func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, b } func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { - if desc.MediaType != ocispec.MediaTypeArtifactManifest && desc.MediaType != ocispec.MediaTypeImageManifest { + var subject ocispec.Descriptor + var err error + switch desc.MediaType { + case ocispec.MediaTypeArtifactManifest: + var artifact ocispec.Artifact + if err := decodeJSON(r, desc, &artifact); err != nil { + return err // TODO: format + } + if artifact.Subject == nil { + return nil + } + subject = *artifact.Subject + desc.ArtifactType = artifact.ArtifactType + desc.Annotations = artifact.Annotations + case ocispec.MediaTypeImageManifest: + var manifest ocispec.Manifest + if err := decodeJSON(r, desc, &manifest); err != nil { + return err // TODO: format + } + if manifest.Subject == nil { + return nil + } + subject = *manifest.Subject + desc.ArtifactType = manifest.Config.MediaType + desc.Annotations = manifest.Annotations + default: // no need to index return nil } - manifestJSON, err := content.ReadAll(r, desc) - if err != nil { - return err - } - type manifest struct { - ArtifactType string `json:"artifactType"` - Subject *ocispec.Descriptor `json:"subject,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - } - var m manifest - if err := json.Unmarshal(manifestJSON, &m); err != nil { - return err - } - if m.Subject == nil { + if state := s.repo.loadReferrersState(); state == referrersStateSupported { // no need to index return nil } - - subject := *m.Subject - // populate ArtifactType and Annotations - desc.ArtifactType = m.ArtifactType - desc.Annotations = m.Annotations ok, err := s.repo.testReferrersAPI(ctx, subject) if err != nil { return err } if ok { // no need to index + s.repo.SetReferrersCapability(true) return nil } + s.repo.SetReferrersCapability(false) + referrersTag := buildReferrersTag(subject) + s.repo.lockReferrersTag(referrersTag) + defer s.repo.unlockReferrersTag(referrersTag) + + // TODO: add referrers or delete referrers var existingReferrers []ocispec.Descriptor // call referrersTagSchema err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { @@ -960,8 +992,6 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip return err // TODO: format } indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) - referrersTag := buildReferrersTag(subject) - // TODO: concurrency? return s.push(ctx, indexDesc, bytes.NewReader(indexJSON), referrersTag) } From 38bdc18f77f900f6a02fbae9fb30f7dbfb464f53 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Thu, 20 Oct 2022 16:16:58 +0800 Subject: [PATCH 04/17] add tests Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 1 + registry/remote/repository_test.go | 304 +++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 710cfada..ea3a8ee9 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -959,6 +959,7 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip defer s.repo.unlockReferrersTag(referrersTag) // TODO: add referrers or delete referrers + // TODO: func() referrers var existingReferrers []ocispec.Descriptor // call referrersTagSchema err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index 188a1f0b..adaece6e 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -35,6 +35,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/internal/interfaces" "oras.land/oras-go/v2/registry" @@ -2767,6 +2768,309 @@ func Test_ManifestStore_Push(t *testing.T) { } } +func Test_ManifestStore_Push_ServerToIndexReferrers(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) + artifact := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + Subject: &subjectDesc, + } + artifactJSON, err := json.Marshal(artifact) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + artifactDesc := content.NewDescriptorFromBytes(artifact.MediaType, artifactJSON) + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Subject: &subjectDesc, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + + var gotManifest []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+artifactDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != artifactDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", artifactDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + result := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{}, + } + if err := json.NewEncoder(w).Encode(result); err != nil { + t.Errorf("failed to write response: %v", err) + } + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + // test push artifact with subject + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, artifactDesc, bytes.NewReader(artifactJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, artifactJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(artifactJSON)) + } + + // test push image manifest with subject + if state := repo.loadReferrersState(); state != referrersStateSupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateSupported) + } + err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } +} + +func Test_ManifestStore_Push_ClientToIndexReferrers(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) + refTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) + artifact := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + Subject: &subjectDesc, + ArtifactType: "application/vnd.test", + Annotations: map[string]string{"foo": "bar"}, + } + artifactJSON, err := json.Marshal(artifact) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + artifactDesc := content.NewDescriptorFromBytes(artifact.MediaType, artifactJSON) + artifactDesc.ArtifactType = artifact.ArtifactType + artifactDesc.Annotations = artifact.Annotations + + // test push artifact with subject + index_1 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + artifactDesc, + }, + } + indexJSON_1, err := json.Marshal(index_1) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + indexDesc_1 := content.NewDescriptorFromBytes(index_1.MediaType, indexJSON_1) + var gotManifest []byte + var gotReferrerIndex []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+artifactDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != artifactDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", artifactDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_1.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, artifactDesc, bytes.NewReader(artifactJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, artifactJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(artifactJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_1) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_1)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + + // test push image manifest with subject + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Config: ocispec.Descriptor{ + MediaType: "testconfig", + }, + Subject: &subjectDesc, + Annotations: map[string]string{"foo": "bar"}, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + manifestDesc.ArtifactType = manifest.Config.MediaType + manifestDesc.Annotations = manifest.Annotations + index_2 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + artifactDesc, + manifestDesc, + }, + } + indexJSON_2, err := json.Marshal(index_2) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + if err := json.NewEncoder(w).Encode(index_1); err != nil { + t.Errorf("failed to write response: %v", err) + } + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } +} + func Test_ManifestStore_Exists(t *testing.T) { manifest := []byte(`{"layers":[]}`) manifestDesc := ocispec.Descriptor{ From b531f9bcb5a54fa595b4ec84a9c56f075fde3117 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Thu, 20 Oct 2022 16:29:06 +0800 Subject: [PATCH 05/17] update test Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 38 +++++++++++------ registry/remote/repository_test.go | 68 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index ea3a8ee9..d1a11717 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -905,10 +905,21 @@ func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, b return err } - return s.indexReferrers(ctx, expected, &buf) + return s.indexReferrers(ctx, expected, &buf, addReferrers) } -func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { +func addReferrers(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor { + var referrers []ocispec.Descriptor + for _, r := range origin { + if r.Digest != desc.Digest { + // TODO: equals? + referrers = append(referrers, r) + } + } + return append(referrers, desc) +} + +func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader, fn func(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor) error { var subject ocispec.Descriptor var err error switch desc.MediaType { @@ -966,27 +977,26 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip existingReferrers = append(existingReferrers, referrers...) return nil }) - if err != nil && !errors.Is(err, errdef.ErrNotFound) { - // If that pull returns a manifest other than the expected image index, the client SHOULD report a failure and skip the remaining steps. + if err != nil { return err } - var referrers []ocispec.Descriptor - // If the tag returns a 404, the client MUST begin with an empty image index. - for _, r := range existingReferrers { - if r.Digest != desc.Digest { - // TODO: equals? - referrers = append(referrers, r) - } - } - referrers = append(referrers, desc) + // var referrers []ocispec.Descriptor + // // If the tag returns a 404, the client MUST begin with an empty image index. + // for _, r := range existingReferrers { + // if r.Digest != desc.Digest { + // // TODO: equals? + // referrers = append(referrers, r) + // } + // } + // referrers = append(referrers, desc) index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version }, MediaType: ocispec.MediaTypeImageIndex, - Manifests: referrers, + Manifests: fn(existingReferrers, desc), } indexJSON, err := json.Marshal(index) if err != nil { diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index adaece6e..4e9850f5 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -3069,6 +3069,74 @@ func Test_ManifestStore_Push_ClientToIndexReferrers(t *testing.T) { if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } + + // test push image manifest with subject again + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + if err := json.NewEncoder(w).Encode(index_2); err != nil { + t.Errorf("failed to write response: %v", err) + } + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } + // referrers list should not change + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } } func Test_ManifestStore_Exists(t *testing.T) { From 5b560c0eef4f72a5a329966d90f2bec5ba6a82b6 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 21 Oct 2022 11:50:46 +0800 Subject: [PATCH 06/17] draft add referrers refactor Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 200 +++++++++++++++++----------------- 1 file changed, 102 insertions(+), 98 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index d1a11717..5188eaee 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -908,104 +908,6 @@ func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, b return s.indexReferrers(ctx, expected, &buf, addReferrers) } -func addReferrers(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor { - var referrers []ocispec.Descriptor - for _, r := range origin { - if r.Digest != desc.Digest { - // TODO: equals? - referrers = append(referrers, r) - } - } - return append(referrers, desc) -} - -func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader, fn func(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor) error { - var subject ocispec.Descriptor - var err error - switch desc.MediaType { - case ocispec.MediaTypeArtifactManifest: - var artifact ocispec.Artifact - if err := decodeJSON(r, desc, &artifact); err != nil { - return err // TODO: format - } - if artifact.Subject == nil { - return nil - } - subject = *artifact.Subject - desc.ArtifactType = artifact.ArtifactType - desc.Annotations = artifact.Annotations - case ocispec.MediaTypeImageManifest: - var manifest ocispec.Manifest - if err := decodeJSON(r, desc, &manifest); err != nil { - return err // TODO: format - } - if manifest.Subject == nil { - return nil - } - subject = *manifest.Subject - desc.ArtifactType = manifest.Config.MediaType - desc.Annotations = manifest.Annotations - default: - // no need to index - return nil - } - - if state := s.repo.loadReferrersState(); state == referrersStateSupported { - // no need to index - return nil - } - ok, err := s.repo.testReferrersAPI(ctx, subject) - if err != nil { - return err - } - if ok { - // no need to index - s.repo.SetReferrersCapability(true) - return nil - } - s.repo.SetReferrersCapability(false) - - referrersTag := buildReferrersTag(subject) - s.repo.lockReferrersTag(referrersTag) - defer s.repo.unlockReferrersTag(referrersTag) - - // TODO: add referrers or delete referrers - // TODO: func() referrers - var existingReferrers []ocispec.Descriptor - // call referrersTagSchema - err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { - existingReferrers = append(existingReferrers, referrers...) - return nil - }) - if err != nil { - return err - } - - // var referrers []ocispec.Descriptor - // // If the tag returns a 404, the client MUST begin with an empty image index. - // for _, r := range existingReferrers { - // if r.Digest != desc.Digest { - // // TODO: equals? - // referrers = append(referrers, r) - // } - // } - // referrers = append(referrers, desc) - - index := ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value. does not pertain to OCI or docker version - }, - MediaType: ocispec.MediaTypeImageIndex, - Manifests: fn(existingReferrers, desc), - } - indexJSON, err := json.Marshal(index) - if err != nil { - return err // TODO: format - } - indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) - return s.push(ctx, indexDesc, bytes.NewReader(indexJSON), referrersTag) -} - // Exists returns true if the described content exists. func (s *manifestStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { _, err := s.Resolve(ctx, target.Digest.String()) @@ -1177,6 +1079,97 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } +// indexReferrers indexes referrers for image or artifact manifest with the +// subject field. +// Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests-with-subject +func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader, fn func(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor) error { + var subject ocispec.Descriptor + var err error + switch desc.MediaType { + case ocispec.MediaTypeArtifactManifest: + var artifact ocispec.Artifact + if err := decodeJSON(r, desc, &artifact); err != nil { + return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) + } + if artifact.Subject == nil { + return nil + } + subject = *artifact.Subject + desc.ArtifactType = artifact.ArtifactType + desc.Annotations = artifact.Annotations + case ocispec.MediaTypeImageManifest: + var manifest ocispec.Manifest + if err := decodeJSON(r, desc, &manifest); err != nil { + return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) + } + if manifest.Subject == nil { + return nil + } + subject = *manifest.Subject + desc.ArtifactType = manifest.Config.MediaType + desc.Annotations = manifest.Annotations + default: + // no subject, no client-side indexing needed + return nil + } + + if state := s.repo.loadReferrersState(); state == referrersStateSupported { + // referrers API is available, no client-side indexing needed + return nil + } + ok, err := s.repo.testReferrersAPI(ctx, subject) + if err != nil { + return err + } + if ok { + // referrers API is available, no client-side indexing needed + s.repo.SetReferrersCapability(true) + return nil + } + s.repo.SetReferrersCapability(false) + + // there can be multiple go-routines updating the referrers tag concurrently + referrersTag := buildReferrersTag(subject) + s.repo.lockReferrersTag(referrersTag) + defer s.repo.unlockReferrersTag(referrersTag) + + // TODO: add referrers or delete referrers + // TODO: func() referrers + var existingReferrers []ocispec.Descriptor + err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { + existingReferrers = append(existingReferrers, referrers...) + return nil + }) + if err != nil { + return err + } + + // var referrers []ocispec.Descriptor + // // If the tag returns a 404, the client MUST begin with an empty image index. + // for _, r := range existingReferrers { + // if r.Digest != desc.Digest { + // // TODO: equals? + // referrers = append(referrers, r) + // } + // } + // referrers = append(referrers, desc) + + referrers := fn(existingReferrers, desc) + index := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: referrers, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return fmt.Errorf("failed to marshal referrers index for referrers tag %s: %w", referrersTag, err) + } + indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) + return s.push(ctx, indexDesc, bytes.NewReader(indexJSON), referrersTag) +} + func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descriptor) (bool, error) { ref := r.Reference ref.Reference = desc.Digest.String() @@ -1203,6 +1196,17 @@ func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descript } } +func addReferrers(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor { + var referrers []ocispec.Descriptor + for _, r := range origin { + if r.Digest != desc.Digest { + // TODO: equals? + referrers = append(referrers, r) + } + } + return append(referrers, desc) +} + // ParseReference parses a reference to a fully qualified reference. func (s *manifestStore) ParseReference(reference string) (registry.Reference, error) { return s.repo.ParseReference(reference) From 9224fc24cbf54e5e98b72fd425e258af87f5ffbe Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 21 Oct 2022 16:14:31 +0800 Subject: [PATCH 07/17] push manfest refactor Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 77 +++++++++++++++++------------------ 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 5188eaee..bdc8cd22 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -905,7 +905,7 @@ func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, b return err } - return s.indexReferrers(ctx, expected, &buf, addReferrers) + return s.indexReferrersForPush(ctx, expected, &buf) } // Exists returns true if the described content exists. @@ -1021,6 +1021,14 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc } // TODO: clone body return s.push(ctx, expected, content, ref.Reference) + // var buf bytes.Buffer + // lr := limitReader(content, s.repo.MaxMetadataBytes) + // tr := io.TeeReader(lr, &buf) + // if err := s.push(ctx, expected, tr, ref.Reference); err != nil { + // return err + // } + + // return s.indexReferrers(ctx, expected, &buf, false) } // push pushes the manifest content, matching the expected descriptor. @@ -1079,12 +1087,11 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } -// indexReferrers indexes referrers for image or artifact manifest with the +// indexReferrersForPush indexes referrers for image or artifact manifest with the // subject field. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests-with-subject -func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, r io.Reader, fn func(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor) error { +func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { var subject ocispec.Descriptor - var err error switch desc.MediaType { case ocispec.MediaTypeArtifactManifest: var artifact ocispec.Artifact @@ -1109,52 +1116,44 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip desc.ArtifactType = manifest.Config.MediaType desc.Annotations = manifest.Annotations default: - // no subject, no client-side indexing needed return nil } - if state := s.repo.loadReferrersState(); state == referrersStateSupported { - // referrers API is available, no client-side indexing needed - return nil - } ok, err := s.repo.testReferrersAPI(ctx, subject) if err != nil { return err } if ok { // referrers API is available, no client-side indexing needed - s.repo.SetReferrersCapability(true) return nil } - s.repo.SetReferrersCapability(false) // there can be multiple go-routines updating the referrers tag concurrently referrersTag := buildReferrersTag(subject) s.repo.lockReferrersTag(referrersTag) defer s.repo.unlockReferrersTag(referrersTag) - // TODO: add referrers or delete referrers - // TODO: func() referrers - var existingReferrers []ocispec.Descriptor + var updatedReferrers []ocispec.Descriptor err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { - existingReferrers = append(existingReferrers, referrers...) + for _, r := range referrers { + if !content.Equal(r, desc) { + updatedReferrers = append(updatedReferrers, r) + } + } return nil }) - if err != nil { + if err != nil && !errors.Is(err, errdef.ErrNotFound) { return err } + updatedReferrers = append(updatedReferrers, desc) + newIndexDesc, newIndex, err := generateReferrersIndex(updatedReferrers) + if err != nil { + return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) + } + return s.push(ctx, newIndexDesc, newIndex, referrersTag) +} - // var referrers []ocispec.Descriptor - // // If the tag returns a 404, the client MUST begin with an empty image index. - // for _, r := range existingReferrers { - // if r.Digest != desc.Digest { - // // TODO: equals? - // referrers = append(referrers, r) - // } - // } - // referrers = append(referrers, desc) - - referrers := fn(existingReferrers, desc) +func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, io.Reader, error) { index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version @@ -1164,13 +1163,20 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip } indexJSON, err := json.Marshal(index) if err != nil { - return fmt.Errorf("failed to marshal referrers index for referrers tag %s: %w", referrersTag, err) + return ocispec.Descriptor{}, nil, err } indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) - return s.push(ctx, indexDesc, bytes.NewReader(indexJSON), referrersTag) + return indexDesc, bytes.NewReader(indexJSON), nil } func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descriptor) (bool, error) { + switch r.loadReferrersState() { + case referrersStateSupported: + return true, nil + case referrersStateUnsupported: + return false, nil + } + ref := r.Reference ref.Reference = desc.Digest.String() ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull) @@ -1188,25 +1194,16 @@ func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descript switch resp.StatusCode { case http.StatusOK: + r.SetReferrersCapability(true) return true, nil case http.StatusNotFound: + r.SetReferrersCapability(false) return false, nil default: return false, errutil.ParseErrorResponse(resp) } } -func addReferrers(origin []ocispec.Descriptor, desc ocispec.Descriptor) []ocispec.Descriptor { - var referrers []ocispec.Descriptor - for _, r := range origin { - if r.Digest != desc.Digest { - // TODO: equals? - referrers = append(referrers, r) - } - } - return append(referrers, desc) -} - // ParseReference parses a reference to a fully qualified reference. func (s *manifestStore) ParseReference(reference string) (registry.Reference, error) { return s.repo.ParseReference(reference) From e8d6780fd1c6167aef015ece52edadd472a9599f Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 21 Oct 2022 16:40:46 +0800 Subject: [PATCH 08/17] refactor manifests.push Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 27 ++++++++++++++++++++------- registry/remote/repository_test.go | 4 ++-- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index bdc8cd22..2412d90e 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -120,15 +120,18 @@ type Repository struct { // default: referrersStateUnknown referrersState referrersState + // referrersTagLocks maps a referrers tag to a lock. referrersTagLocks sync.Map // map[string]sync.Mutex } +// lockReferrersTag locks a referrers tag. func (r *Repository) lockReferrersTag(tag string) { v, _ := r.referrersTagLocks.LoadOrStore(tag, &sync.Mutex{}) lock := v.(*sync.Mutex) lock.Lock() } +// unlockReferrersTag unlocks a referrers tag. func (r *Repository) unlockReferrersTag(tag string) { v, ok := r.referrersTagLocks.Load(tag) if !ok { @@ -897,9 +900,10 @@ func (s *manifestStore) Fetch(ctx context.Context, target ocispec.Descriptor) (r } // Push pushes the content, matching the expected descriptor. -func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, body io.Reader) error { +func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + // copy content for referrers indexing var buf bytes.Buffer - lr := limitReader(body, s.repo.MaxMetadataBytes) + lr := limitReader(content, s.repo.MaxMetadataBytes) tr := io.TeeReader(lr, &buf) if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { return err @@ -1019,8 +1023,9 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc if err != nil { return err } - // TODO: clone body + return s.push(ctx, expected, content, ref.Reference) + // // copy content for referrers indexing // var buf bytes.Buffer // lr := limitReader(content, s.repo.MaxMetadataBytes) // tr := io.TeeReader(lr, &buf) @@ -1028,7 +1033,7 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc // return err // } - // return s.indexReferrers(ctx, expected, &buf, false) + // return s.indexReferrersForPush(ctx, expected, &buf) } // push pushes the manifest content, matching the expected descriptor. @@ -1099,6 +1104,7 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) } if artifact.Subject == nil { + // no subject, no indexing needed return nil } subject = *artifact.Subject @@ -1110,6 +1116,7 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) } if manifest.Subject == nil { + // no subject, no indexing needed return nil } subject = *manifest.Subject @@ -1119,11 +1126,11 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. return nil } - ok, err := s.repo.testReferrersAPI(ctx, subject) + yes, err := s.repo.isReferrersAPIAvailable(ctx, subject) if err != nil { return err } - if ok { + if yes { // referrers API is available, no client-side indexing needed return nil } @@ -1136,6 +1143,7 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. var updatedReferrers []ocispec.Descriptor err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { for _, r := range referrers { + // skip duplicate entry if !content.Equal(r, desc) { updatedReferrers = append(updatedReferrers, r) } @@ -1145,6 +1153,7 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. if err != nil && !errors.Is(err, errdef.ErrNotFound) { return err } + updatedReferrers = append(updatedReferrers, desc) newIndexDesc, newIndex, err := generateReferrersIndex(updatedReferrers) if err != nil { @@ -1153,6 +1162,8 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. return s.push(ctx, newIndexDesc, newIndex, referrersTag) } +// generateReferrersIndex generates an image index containing the given +// referrers list in its manifests field. func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, io.Reader, error) { index := ocispec.Index{ Versioned: specs.Versioned{ @@ -1169,7 +1180,8 @@ func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, return indexDesc, bytes.NewReader(indexJSON), nil } -func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descriptor) (bool, error) { +// isReferrersAPIAvailable returns true if the Referrers API is available for r. +func (r *Repository) isReferrersAPIAvailable(ctx context.Context, desc ocispec.Descriptor) (bool, error) { switch r.loadReferrersState() { case referrersStateSupported: return true, nil @@ -1177,6 +1189,7 @@ func (r *Repository) testReferrersAPI(ctx context.Context, desc ocispec.Descript return false, nil } + // referrers state is unknown ref := r.Reference ref.Reference = desc.Digest.String() ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull) diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index 4e9850f5..6cf0f94d 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -2768,7 +2768,7 @@ func Test_ManifestStore_Push(t *testing.T) { } } -func Test_ManifestStore_Push_ServerToIndexReferrers(t *testing.T) { +func Test_ManifestStore_Push_ReferrersAPIAvailable(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`) subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) @@ -2872,7 +2872,7 @@ func Test_ManifestStore_Push_ServerToIndexReferrers(t *testing.T) { } } -func Test_ManifestStore_Push_ClientToIndexReferrers(t *testing.T) { +func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`) subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) From b358e87fff9009b43cd2d543af520bfa44704a9a Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 21 Oct 2022 16:42:55 +0800 Subject: [PATCH 09/17] minor adjustment Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 43 ++++++++++++++--------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 2412d90e..b0acfcad 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -124,23 +124,6 @@ type Repository struct { referrersTagLocks sync.Map // map[string]sync.Mutex } -// lockReferrersTag locks a referrers tag. -func (r *Repository) lockReferrersTag(tag string) { - v, _ := r.referrersTagLocks.LoadOrStore(tag, &sync.Mutex{}) - lock := v.(*sync.Mutex) - lock.Lock() -} - -// unlockReferrersTag unlocks a referrers tag. -func (r *Repository) unlockReferrersTag(tag string) { - v, ok := r.referrersTagLocks.Load(tag) - if !ok { - return - } - lock := v.(*sync.Mutex) - lock.Unlock() -} - // NewRepository creates a client to the remote repository identified by a // reference. // Example: localhost:5000/hello-world @@ -188,6 +171,23 @@ func (r *Repository) loadReferrersState() referrersState { return atomic.LoadInt32(&r.referrersState) } +// lockReferrersTag locks a referrers tag. +func (r *Repository) lockReferrersTag(tag string) { + v, _ := r.referrersTagLocks.LoadOrStore(tag, &sync.Mutex{}) + lock := v.(*sync.Mutex) + lock.Lock() +} + +// unlockReferrersTag unlocks a referrers tag. +func (r *Repository) unlockReferrersTag(tag string) { + v, ok := r.referrersTagLocks.Load(tag) + if !ok { + return + } + lock := v.(*sync.Mutex) + lock.Unlock() +} + // client returns an HTTP client used to access the remote repository. // A default HTTP client is return if the client is not configured. func (r *Repository) client() Client { @@ -1025,15 +1025,6 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc } return s.push(ctx, expected, content, ref.Reference) - // // copy content for referrers indexing - // var buf bytes.Buffer - // lr := limitReader(content, s.repo.MaxMetadataBytes) - // tr := io.TeeReader(lr, &buf) - // if err := s.push(ctx, expected, tr, ref.Reference); err != nil { - // return err - // } - - // return s.indexReferrersForPush(ctx, expected, &buf) } // push pushes the manifest content, matching the expected descriptor. From 1202773fca485b0f6ba36997d3e7299f2132b8f5 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 21 Oct 2022 17:59:11 +0800 Subject: [PATCH 10/17] clean up Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/example_test.go | 69 ++-- registry/remote/repository.go | 52 ++- registry/remote/repository_test.go | 640 ++++++++++++++++++++++++++++- 3 files changed, 691 insertions(+), 70 deletions(-) diff --git a/registry/remote/example_test.go b/registry/remote/example_test.go index 9dc19a78..55d1043d 100644 --- a/registry/remote/example_test.go +++ b/registry/remote/example_test.go @@ -40,7 +40,7 @@ import ( const ( exampleRepositoryName = "example" exampleTag = "latest" - exampleManifest = "Example manifest content" + exampleConfig = "Example config content" exampleLayer = "Example layer content" exampleUploadUUid = "0bc84d80-837c-41d9-824e-1907463c53b3" ManifestDigest = "sha256:0b696106ecd0654e031f19e0a8cbd1aee4ad457d7c9cea881f07b12a930cd307" @@ -49,12 +49,19 @@ const ( ) var ( - exampleLayerDigest = digest.FromBytes([]byte(exampleLayer)).String() - exampleManifestDigest = digest.FromBytes([]byte(exampleManifest)).String() - exampleManifestDescriptor = ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, - Digest: digest.Digest(exampleManifestDigest), - Size: int64(len(exampleManifest))} + exampleLayerDescriptor = content.NewDescriptorFromBytes(ocispec.MediaTypeImageLayer, []byte(exampleLayer)) + exampleLayerDigest = exampleLayerDescriptor.Digest.String() + exampleManifest, _ = json.Marshal(ocispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + Config: content.NewDescriptorFromBytes(ocispec.MediaTypeImageConfig, []byte(exampleConfig)), + Layers: []ocispec.Descriptor{ + exampleLayerDescriptor, + }, + }) + exampleManifestDescriptor = content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, exampleManifest) + exampleManifestDigest = exampleManifestDescriptor.Digest.String() exampleSignatureManifest, _ = json.Marshal(ocispec.Artifact{ MediaType: ocispec.MediaTypeArtifactManifest, ArtifactType: "example/signature", @@ -322,8 +329,8 @@ func ExampleRepository_Resolve_byTag() { // Output: // application/vnd.oci.image.manifest.v1+json - // sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b - // 24 + // sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7 + // 337 } // ExampleRepository_Resolve_byDigest gives example snippets for resolving a digest to a manifest descriptor. @@ -333,7 +340,7 @@ func ExampleRepository_Resolve_byDigest() { panic(err) } ctx := context.Background() - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" descriptor, err := repo.Resolve(ctx, exampleDigest) if err != nil { panic(err) @@ -345,8 +352,8 @@ func ExampleRepository_Resolve_byDigest() { // Output: // application/vnd.oci.image.manifest.v1+json - // sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b - // 24 + // sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7 + // 337 } // ExampleRepository_Fetch_byTag gives example snippets for downloading a manifest by tag. @@ -375,7 +382,7 @@ func ExampleRepository_Fetch_manifestByTag() { fmt.Println(string(pulledBlob)) // Output: - // Example manifest content + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } // ExampleRepository_Fetch_manifestByDigest gives example snippets for downloading a manifest by digest. @@ -386,7 +393,7 @@ func ExampleRepository_Fetch_manifestByDigest() { } ctx := context.Background() - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" // resolve the blob descriptor to obtain the size of the blob descriptor, err := repo.Resolve(ctx, exampleDigest) if err != nil { @@ -404,7 +411,7 @@ func ExampleRepository_Fetch_manifestByDigest() { fmt.Println(string(pulled)) // Output: - // Example manifest content + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } // ExampleRepository_Fetch_artifactReferenceManifest gives an example of fetching @@ -443,8 +450,8 @@ func ExampleRepository_Fetch_artifactReferenceManifest() { panic(err) } // Output: - // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/SBoM","subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b","size":24}} - // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/signature","subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b","size":24}} + // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/SBoM","subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7","size":337}} + // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/signature","subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7","size":337}} } // ExampleRepository_fetchArtifactBlobs gives an example of pulling the blobs @@ -457,7 +464,7 @@ func ExampleRepository_fetchArtifactBlobs() { ctx := context.Background() // 1. Fetch the artifact manifest by digest. - exampleDigest := "sha256:1907bb31b7add4d47d74d2c5c1c10d67b757a996f8e8186e562113bc9879b1a3" + exampleDigest := "sha256:f3550fd0947402d140fd0470702abc92c69f7e9b08d5ca2438f42f8a0ea3fd97" descriptor, rc, err := repo.FetchReference(ctx, exampleDigest) if err != nil { panic(err) @@ -484,7 +491,7 @@ func ExampleRepository_fetchArtifactBlobs() { } // Output: - // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/manifest","blobs":[{"mediaType":"application/tar","digest":"sha256:8d6497c94694a292c04f85cd055d8b5c03eda835dd311e20dfbbf029ff9748cc","size":20}],"subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b","size":24}} + // {"mediaType":"application/vnd.oci.artifact.manifest.v1+json","artifactType":"example/manifest","blobs":[{"mediaType":"application/tar","digest":"sha256:8d6497c94694a292c04f85cd055d8b5c03eda835dd311e20dfbbf029ff9748cc","size":20}],"subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7","size":337}} // example blob content } @@ -510,7 +517,7 @@ func ExampleRepository_FetchReference_manifestByTag() { fmt.Println(string(pulledBlob)) // Output: - // Example manifest content + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } // ExampleRepository_FetchReference_manifestByDigest gives example snippets for downloading a manifest by digest. @@ -521,7 +528,7 @@ func ExampleRepository_FetchReference_manifestByDigest() { } ctx := context.Background() - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" descriptor, rc, err := repo.FetchReference(ctx, exampleDigest) if err != nil { panic(err) @@ -535,7 +542,7 @@ func ExampleRepository_FetchReference_manifestByDigest() { fmt.Println(string(pulled)) // Output: - // Example manifest content + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } // ExampleRepository_Fetch_layer gives example snippets for downloading a layer blob by digest. @@ -593,7 +600,7 @@ func ExampleRepository_Tag() { } ctx := context.Background() - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" descriptor, err := repo.Resolve(ctx, exampleDigest) if err != nil { panic(err) @@ -623,7 +630,7 @@ func ExampleRepository_TagReference() { } // tag a manifest referenced by the exampleDigest below - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" tag := "latest" err = oras.Tag(ctx, repo, exampleDigest, tag) if err != nil { @@ -685,9 +692,9 @@ func Example_pullByTag() { fmt.Println(string(pulledBlob)) // Output: - // sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b - // 24 - // Example manifest content + // sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7 + // 337 + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } func Example_pullByDigest() { @@ -697,7 +704,7 @@ func Example_pullByDigest() { } ctx := context.Background() - exampleDigest := "sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b" + exampleDigest := "sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7" // 1. resolve the descriptor descriptor, err := repo.Resolve(ctx, exampleDigest) if err != nil { @@ -714,9 +721,9 @@ func Example_pullByDigest() { fmt.Println(string(pulledBlob)) // Output: - // sha256:00e5ffa7d914b4e6aa3f1a324f37df0625ccc400be333deea5ecaa199f9eff5b - // 24 - // Example manifest content + // sha256:b53dc03a49f383ba230d8ac2b78a9c4aec132e4a9f36cc96524df98163202cc7 + // 337 + // {"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:569224ae188c06e97b9fcadaeb2358fb0fb7c4eb105d49aee2620b2719abea43","size":22},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:ef79e47691ad1bc702d7a256da6323ec369a8fc3159b4f1798a47136f3b38c10","size":21}]} } // Example_pushAndTag gives example snippet of pushing an OCI image with a tag. diff --git a/registry/remote/repository.go b/registry/remote/repository.go index b0acfcad..da4599c0 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -1024,7 +1024,14 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc return err } - return s.push(ctx, expected, content, ref.Reference) + // copy content for referrers indexing + var buf bytes.Buffer + lr := limitReader(content, s.repo.MaxMetadataBytes) + tr := io.TeeReader(lr, &buf) + if err := s.push(ctx, expected, tr, ref.Reference); err != nil { + return err + } + return s.indexReferrersForPush(ctx, expected, &buf) } // push pushes the manifest content, matching the expected descriptor. @@ -1146,29 +1153,11 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. } updatedReferrers = append(updatedReferrers, desc) - newIndexDesc, newIndex, err := generateReferrersIndex(updatedReferrers) + indexDesc, index, err := generateReferrersIndex(updatedReferrers) if err != nil { return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) } - return s.push(ctx, newIndexDesc, newIndex, referrersTag) -} - -// generateReferrersIndex generates an image index containing the given -// referrers list in its manifests field. -func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, io.Reader, error) { - index := ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value. does not pertain to OCI or docker version - }, - MediaType: ocispec.MediaTypeImageIndex, - Manifests: referrers, - } - indexJSON, err := json.Marshal(index) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) - return indexDesc, bytes.NewReader(indexJSON), nil + return s.push(ctx, indexDesc, bytes.NewReader(index), referrersTag) } // isReferrersAPIAvailable returns true if the Referrers API is available for r. @@ -1347,3 +1336,24 @@ func verifyContentDigest(resp *http.Response, expected digest.Digest) error { return nil } + +// generateReferrersIndex generates an image index containing the given +// referrers list in its manifests field. +func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, []byte, error) { + if referrers == nil { + referrers = []ocispec.Descriptor{} // make it an empty array to prevent potential server-side bugs + } + index := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: referrers, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + indexDesc := content.NewDescriptorFromBytes(index.MediaType, indexJSON) + return indexDesc, indexJSON, nil +} diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index 6cf0f94d..f5fc6357 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -33,7 +33,7 @@ import ( "testing" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" + specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" @@ -1017,8 +1017,8 @@ func TestRepository_Referrers(t *testing.T) { ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { path := "/v2/test/referrers/" + manifestDesc.Digest.String() if r.Method != http.MethodGet || r.URL.Path != path { - refTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) - if r.URL.Path != "/v2/test/manifests/"+refTag { + referrersTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) + if r.URL.Path != "/v2/test/manifests/"+referrersTag { t.Errorf("unexpected access: %s %q", r.Method, r.URL) } w.WriteHeader(http.StatusNotFound) @@ -1183,8 +1183,8 @@ func TestRepository_Referrers_TagSchemaFallback(t *testing.T) { }, } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - refTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) - path := "/v2/test/manifests/" + refTag + referrersTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) + path := "/v2/test/manifests/" + referrersTag if r.Method != http.MethodGet || r.URL.Path != path { if r.URL.Path != "/v2/test/referrers/"+manifestDesc.Digest.String() { t.Errorf("unexpected access: %s %q", r.Method, r.URL) @@ -1287,8 +1287,8 @@ func TestRepository_Referrers_TagSchemaFallback_NotFound(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { referrersUrl := "/v2/test/referrers/" + manifestDesc.Digest.String() - refTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) - tagSchemaUrl := "/v2/test/manifests/" + refTag + referrersTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) + tagSchemaUrl := "/v2/test/manifests/" + referrersTag if r.Method == http.MethodGet || r.URL.Path == referrersUrl || r.URL.Path == tagSchemaUrl { @@ -1354,8 +1354,8 @@ func TestRepository_Referrers_BadRequest(t *testing.T) { } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { referrersUrl := "/v2/test/referrers/" + manifestDesc.Digest.String() - refTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) - tagSchemaUrl := "/v2/test/manifests/" + refTag + referrersTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) + tagSchemaUrl := "/v2/test/manifests/" + referrersTag if r.Method == http.MethodGet || r.URL.Path == referrersUrl || r.URL.Path == tagSchemaUrl { @@ -1751,8 +1751,8 @@ func TestRepository_Referrers_TagSchemaFallback_ClientFiltering(t *testing.T) { } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - refTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) - path := "/v2/test/manifests/" + refTag + referrersTag := strings.Replace(manifestDesc.Digest.String(), ":", "-", 1) + path := "/v2/test/manifests/" + referrersTag if r.Method != http.MethodGet || r.URL.Path != path { if r.URL.Path != "/v2/test/referrers/"+manifestDesc.Digest.String() { t.Errorf("unexpected access: %s %q", r.Method, r.URL) @@ -2876,7 +2876,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`) subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) - refTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) + referrersTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) artifact := ocispec.Artifact{ MediaType: ocispec.MediaTypeArtifactManifest, Subject: &subjectDesc, @@ -2924,9 +2924,9 @@ func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { w.WriteHeader(http.StatusCreated) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) - case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: w.WriteHeader(http.StatusNotFound) - case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) break @@ -3020,11 +3020,11 @@ func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { w.WriteHeader(http.StatusCreated) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) - case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: if err := json.NewEncoder(w).Encode(index_1); err != nil { t.Errorf("failed to write response: %v", err) } - case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) break @@ -3087,11 +3087,11 @@ func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { w.WriteHeader(http.StatusCreated) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) - case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: if err := json.NewEncoder(w).Encode(index_2); err != nil { t.Errorf("failed to write response: %v", err) } - case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+refTag: + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) break @@ -3627,6 +3627,383 @@ func Test_ManifestStore_PushReference(t *testing.T) { } } +func Test_ManifestStore_PushReference_ReferrersAPIAvailable(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) + artifact := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + Subject: &subjectDesc, + } + artifactJSON, err := json.Marshal(artifact) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + artifactDesc := content.NewDescriptorFromBytes(artifact.MediaType, artifactJSON) + artifactRef := "foo" + + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Subject: &subjectDesc, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + manifestRef := "bar" + + var gotManifest []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+artifactRef: + if contentType := r.Header.Get("Content-Type"); contentType != artifactDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", artifactDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestRef: + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + result := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{}, + } + if err := json.NewEncoder(w).Encode(result); err != nil { + t.Errorf("failed to write response: %v", err) + } + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + // test push artifact with subject + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.PushReference(ctx, artifactDesc, bytes.NewReader(artifactJSON), artifactRef) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, artifactJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(artifactJSON)) + } + + // test push image manifest with subject + if state := repo.loadReferrersState(); state != referrersStateSupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateSupported) + } + err = repo.PushReference(ctx, manifestDesc, bytes.NewReader(manifestJSON), manifestRef) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } +} + +func Test_ManifestStore_PushReference_ReferrersAPIUnAvailable(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) + referrersTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) + artifact := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + Subject: &subjectDesc, + ArtifactType: "application/vnd.test", + Annotations: map[string]string{"foo": "bar"}, + } + artifactJSON, err := json.Marshal(artifact) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + artifactDesc := content.NewDescriptorFromBytes(artifact.MediaType, artifactJSON) + artifactDesc.ArtifactType = artifact.ArtifactType + artifactDesc.Annotations = artifact.Annotations + artifactRef := "foo" + + // test push artifact with subject + index_1 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + artifactDesc, + }, + } + indexJSON_1, err := json.Marshal(index_1) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + indexDesc_1 := content.NewDescriptorFromBytes(index_1.MediaType, indexJSON_1) + var gotManifest []byte + var gotReferrerIndex []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+artifactRef: + if contentType := r.Header.Get("Content-Type"); contentType != artifactDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", artifactDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_1.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.PushReference(ctx, artifactDesc, bytes.NewReader(artifactJSON), artifactRef) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, artifactJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(artifactJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_1) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_1)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + + // test push image manifest with subject + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Config: ocispec.Descriptor{ + MediaType: "testconfig", + }, + Subject: &subjectDesc, + Annotations: map[string]string{"foo": "bar"}, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + manifestDesc.ArtifactType = manifest.Config.MediaType + manifestDesc.Annotations = manifest.Annotations + manifestRef := "bar" + + index_2 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + artifactDesc, + manifestDesc, + }, + } + indexJSON_2, err := json.Marshal(index_2) + if err != nil { + t.Errorf("failed to marshal manifest: %v", err) + } + indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestRef: + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if err := json.NewEncoder(w).Encode(index_1); err != nil { + t.Errorf("failed to write response: %v", err) + } + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.PushReference(ctx, manifestDesc, bytes.NewReader(manifestJSON), manifestRef) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + + // test push image manifest with subject again + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if err := json.NewEncoder(w).Encode(index_2); err != nil { + t.Errorf("failed to write response: %v", err) + } + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } + // referrers list should not change + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } +} + func Test_ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders(t *testing.T) { reference := registry.Reference{ Registry: "eastern.haan.com", @@ -4382,3 +4759,230 @@ func Test_getReferrersTag(t *testing.T) { }) } } + +func Test_generateReferrersIndex(t *testing.T) { + referrer_1 := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + ArtifactType: "foo", + } + referrerJSON_1, err := json.Marshal(referrer_1) + if err != nil { + t.Fatal("failed to marshal manifest:", err) + } + referrer_2 := ocispec.Artifact{ + MediaType: ocispec.MediaTypeArtifactManifest, + ArtifactType: "bar", + } + referrerJSON_2, err := json.Marshal(referrer_2) + if err != nil { + t.Fatal("failed to marshal manifest:", err) + } + referrers := []ocispec.Descriptor{ + content.NewDescriptorFromBytes(referrer_1.MediaType, referrerJSON_1), + content.NewDescriptorFromBytes(referrer_2.MediaType, referrerJSON_2), + } + + wantIndex := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: referrers, + } + wantIndexJSON, err := json.Marshal(wantIndex) + if err != nil { + t.Fatal("failed to marshal index:", err) + } + wantIndexDesc := content.NewDescriptorFromBytes(wantIndex.MediaType, wantIndexJSON) + + wantEmptyIndex := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{}, + } + wantEmptyIndexJSON, err := json.Marshal(wantEmptyIndex) + if err != nil { + t.Fatal("failed to marshal index:", err) + } + wantEmptyIndexDesc := content.NewDescriptorFromBytes(wantEmptyIndex.MediaType, wantEmptyIndexJSON) + + tests := []struct { + name string + referrers []ocispec.Descriptor + wantDesc ocispec.Descriptor + wantBytes []byte + wantErr bool + }{ + { + name: "non-empty referrers list", + referrers: referrers, + wantDesc: wantIndexDesc, + wantBytes: wantIndexJSON, + wantErr: false, + }, + { + name: "nil referrers list", + referrers: nil, + wantDesc: wantEmptyIndexDesc, + wantBytes: wantEmptyIndexJSON, + wantErr: false, + }, + { + name: "empty referrers list", + referrers: nil, + wantDesc: wantEmptyIndexDesc, + wantBytes: wantEmptyIndexJSON, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := generateReferrersIndex(tt.referrers) + if (err != nil) != tt.wantErr { + t.Errorf("generateReferrersIndex() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.wantDesc) { + t.Errorf("generateReferrersIndex() got = %v, want %v", got, tt.wantDesc) + } + if !reflect.DeepEqual(got1, tt.wantBytes) { + t.Errorf("generateReferrersIndex() got1 = %v, want %v", got1, tt.wantBytes) + } + }) + } +} + +func TestRepository_isReferrersAPIAvailable(t *testing.T) { + manifest := []byte(`{"layers":[]}`) + manifestDesc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifest), + Size: int64(len(manifest)), + } + + // referrers available + count := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+manifestDesc.Digest.String(): + count++ + w.WriteHeader(http.StatusOK) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + // 1st call + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + got, err := repo.isReferrersAPIAvailable(ctx, manifestDesc) + if err != nil { + t.Errorf("Repository.isReferrersAPIAvailable() error = %v, wantErr %v", err, nil) + } + if got != true { + t.Errorf("Repository.isReferrersAPIAvailable() = %v, want %v", got, true) + } + if state := repo.loadReferrersState(); state != referrersStateSupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateSupported) + } + if count != 1 { + t.Errorf("count(Repository.isReferrersAPIAvailable()) = %v, want %v", count, 1) + } + + // 2nd call + if state := repo.loadReferrersState(); state != referrersStateSupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateSupported) + } + got, err = repo.isReferrersAPIAvailable(ctx, manifestDesc) + if err != nil { + t.Errorf("Repository.isReferrersAPIAvailable() error = %v, wantErr %v", err, nil) + } + if got != true { + t.Errorf("Repository.isReferrersAPIAvailable() = %v, want %v", got, true) + } + if state := repo.loadReferrersState(); state != referrersStateSupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateSupported) + } + if count != 1 { + t.Errorf("count(Repository.isReferrersAPIAvailable()) = %v, want %v", count, 1) + } + + // referrers unavailable + count = 0 + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+manifestDesc.Digest.String(): + count++ + w.WriteHeader(http.StatusNotFound) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + // 1st call + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + got, err = repo.isReferrersAPIAvailable(ctx, manifestDesc) + if err != nil { + t.Errorf("Repository.isReferrersAPIAvailable() error = %v, wantErr %v", err, nil) + } + if got != false { + t.Errorf("Repository.isReferrersAPIAvailable() = %v, want %v", got, false) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + if count != 1 { + t.Errorf("count(Repository.isReferrersAPIAvailable()) = %v, want %v", count, 1) + } + + // 2nd call + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + got, err = repo.isReferrersAPIAvailable(ctx, manifestDesc) + if err != nil { + t.Errorf("Repository.isReferrersAPIAvailable() error = %v, wantErr %v", err, nil) + } + if got != false { + t.Errorf("Repository.isReferrersAPIAvailable() = %v, want %v", got, false) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + if count != 1 { + t.Errorf("count(Repository.isReferrersAPIAvailable()) = %v, want %v", count, 1) + } +} From 34d3dad0d4f93b8fc0edcc11eec0edfd97630d97 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 15:18:20 +0800 Subject: [PATCH 11/17] fix typo Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index f5fc6357..9637ee2f 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -2872,7 +2872,7 @@ func Test_ManifestStore_Push_ReferrersAPIAvailable(t *testing.T) { } } -func Test_ManifestStore_Push_ReferrersAPIUnAvailable(t *testing.T) { +func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`) subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) @@ -3734,7 +3734,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIAvailable(t *testing.T) { } } -func Test_ManifestStore_PushReference_ReferrersAPIUnAvailable(t *testing.T) { +func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`) subjectDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, subject) From f89000dd0c7139526ee05a596a14ffb121f40eec Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 17:49:39 +0800 Subject: [PATCH 12/17] address comments Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 115 +++++++++++++++++++---------- registry/remote/repository_test.go | 72 +++++++----------- 2 files changed, 105 insertions(+), 82 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index da4599c0..88b07c9e 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -388,7 +388,10 @@ func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, art if state == referrersStateUnsupported { // The repository is known to not support Referrers API, fallback to // referrers tag schema. - return r.referrersByTagSchema(ctx, desc, artifactType, fn) + if _, err := r.referrersByTagSchema(ctx, desc, artifactType, fn); err != nil && !errors.Is(err, errdef.ErrNotFound) { + return err + } + return nil } err := r.referrersByAPI(ctx, desc, artifactType, fn) @@ -403,7 +406,10 @@ func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, art // A 404 returned by Referrers API indicates that Referrers API is // not supported. Fallback to referrers tag schema. r.SetReferrersCapability(false) - return r.referrersByTagSchema(ctx, desc, artifactType, fn) + if _, err := r.referrersByTagSchema(ctx, desc, artifactType, fn); err != nil && !errors.Is(err, errdef.ErrNotFound) { + return err + } + return nil } return err } @@ -485,31 +491,27 @@ func (r *Repository) referrersPageByAPI(ctx context.Context, artifactType string // fn is called for the referrers result. If artifactType is not empty, // only referrers of the same artifact type are fed to fn. // reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#backwards-compatibility -func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error { +func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) (ocispec.Descriptor, error) { referrersTag := buildReferrersTag(desc) desc, rc, err := r.FetchReference(ctx, referrersTag) if err != nil { - if errors.Is(err, errdef.ErrNotFound) { - // no referrers to the manifest - return nil - } - return err + return ocispec.Descriptor{}, err } defer rc.Close() if err := limitSize(desc, r.MaxMetadataBytes); err != nil { - return fmt.Errorf("failed to read referrers index from referrers tag %s: %w", referrersTag, err) + return ocispec.Descriptor{}, fmt.Errorf("failed to read referrers index from referrers tag %s: %w", referrersTag, err) } var index ocispec.Index if err := decodeJSON(rc, desc, &index); err != nil { - return fmt.Errorf("failed to decode referrers index from referrers tag %s: %w", referrersTag, err) + return ocispec.Descriptor{}, fmt.Errorf("failed to decode referrers index from referrers tag %s: %w", referrersTag, err) } referrers := filterReferrers(index.Manifests, artifactType) - if len(referrers) == 0 { - return nil + if err := fn(referrers); err != nil { + return ocispec.Descriptor{}, err } - return fn(referrers) + return desc, nil } // buildReferrersTag builds the referrers tag for the given manifest descriptor. @@ -901,15 +903,24 @@ func (s *manifestStore) Fetch(ctx context.Context, target ocispec.Descriptor) (r // Push pushes the content, matching the expected descriptor. func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { - // copy content for referrers indexing - var buf bytes.Buffer - lr := limitReader(content, s.repo.MaxMetadataBytes) - tr := io.TeeReader(lr, &buf) - if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { - return err - } + switch expected.MediaType { + case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: + if state := s.repo.loadReferrersState(); state == referrersStateSupported { + // referrers API is available, no client-side indexing needed + return s.push(ctx, expected, content, expected.Digest.String()) + } - return s.indexReferrersForPush(ctx, expected, &buf) + // buffer content for referrers indexing + var buf bytes.Buffer + lr := limitReader(content, s.repo.MaxMetadataBytes) + tr := io.TeeReader(lr, &buf) + if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { + return err + } + return s.indexReferrersForPush(ctx, expected, &buf) + default: + return s.push(ctx, expected, content, expected.Digest.String()) + } } // Exists returns true if the described content exists. @@ -1024,14 +1035,24 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc return err } - // copy content for referrers indexing - var buf bytes.Buffer - lr := limitReader(content, s.repo.MaxMetadataBytes) - tr := io.TeeReader(lr, &buf) - if err := s.push(ctx, expected, tr, ref.Reference); err != nil { - return err + switch expected.MediaType { + case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: + if state := s.repo.loadReferrersState(); state == referrersStateSupported { + // referrers API is available, no client-side indexing needed + return s.push(ctx, expected, content, ref.Reference) + } + + // buffer content for referrers indexing + var buf bytes.Buffer + lr := limitReader(content, s.repo.MaxMetadataBytes) + tr := io.TeeReader(lr, &buf) + if err := s.push(ctx, expected, tr, ref.Reference); err != nil { + return err + } + return s.indexReferrersForPush(ctx, expected, &buf) + default: + return s.push(ctx, expected, content, ref.Reference) } - return s.indexReferrersForPush(ctx, expected, &buf) } // push pushes the manifest content, matching the expected descriptor. @@ -1090,8 +1111,8 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } -// indexReferrersForPush indexes referrers for image or artifact manifest with the -// subject field. +// indexReferrersForPush indexes referrers for image or artifact manifest with +// the subject field. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests-with-subject func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { var subject ocispec.Descriptor @@ -1139,25 +1160,43 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. defer s.repo.unlockReferrersTag(referrersTag) var updatedReferrers []ocispec.Descriptor - err = s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { + var skipUpdate, skipDelete bool + oldIndexDesc, err := s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { for _, r := range referrers { - // skip duplicate entry - if !content.Equal(r, desc) { - updatedReferrers = append(updatedReferrers, r) + if content.Equal(r, desc) { + // desc is already in the referrers list, skip update + skipUpdate = true + break } + updatedReferrers = append(updatedReferrers, r) } return nil }) - if err != nil && !errors.Is(err, errdef.ErrNotFound) { - return err + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // there is no old referrers index, skip delete + skipDelete = true + } else { + return err + } + } + if skipUpdate { + return nil } updatedReferrers = append(updatedReferrers, desc) - indexDesc, index, err := generateReferrersIndex(updatedReferrers) + newIndexDesc, newIndex, err := generateReferrersIndex(updatedReferrers) if err != nil { return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) } - return s.push(ctx, indexDesc, bytes.NewReader(index), referrersTag) + if err := s.push(ctx, newIndexDesc, bytes.NewReader(newIndex), referrersTag); err != nil { + return fmt.Errorf("failed to push referrers index tagged by %s: %w", referrersTag, err) + } + + if skipDelete { + return nil + } + return s.repo.delete(ctx, oldIndexDesc, true) } // isReferrersAPIAvailable returns true if the Referrers API is available for r. diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index 9637ee2f..8d96b5a4 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -2973,7 +2973,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test push image manifest with subject + // test push image manifest with subject, referrer list should be updated manifest := ocispec.Manifest{ MediaType: ocispec.MediaTypeImageManifest, Config: ocispec.Descriptor{ @@ -3004,6 +3004,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("failed to marshal manifest: %v", err) } indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) + var manifestDeleted bool ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3021,9 +3022,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if err := json.NewEncoder(w).Encode(index_1); err != nil { - t.Errorf("failed to write response: %v", err) - } + w.Write(indexJSON_1) case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) @@ -3036,6 +3035,10 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { gotReferrerIndex = buf.Bytes() w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_1.Digest.String(): + manifestDeleted = true + // no "Docker-Content-Digest" header for manifest deletion + w.WriteHeader(http.StatusAccepted) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3066,11 +3069,14 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotReferrerIndex, indexJSON_2) { t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } + if !manifestDeleted { + t.Errorf("manifestDeleted = %v, want %v", manifestDeleted, true) + } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test push image manifest with subject again + // test push image manifest with subject again, referrers list should not be changed ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3088,21 +3094,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if err := json.NewEncoder(w).Encode(index_2); err != nil { - t.Errorf("failed to write response: %v", err) - } - case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { - w.WriteHeader(http.StatusBadRequest) - break - } - buf := bytes.NewBuffer(nil) - if _, err := buf.ReadFrom(r.Body); err != nil { - t.Errorf("fail to read: %v", err) - } - gotReferrerIndex = buf.Bytes() - w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) - w.WriteHeader(http.StatusCreated) + w.Write(indexJSON_2) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3130,7 +3122,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotManifest, manifestJSON) { t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) } - // referrers list should not change + // referrers list should not be changed if !bytes.Equal(gotReferrerIndex, indexJSON_2) { t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } @@ -3836,7 +3828,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test push image manifest with subject + // test push image manifest with subject, referrers list should be updated manifest := ocispec.Manifest{ MediaType: ocispec.MediaTypeImageManifest, Config: ocispec.Descriptor{ @@ -3869,6 +3861,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("failed to marshal manifest: %v", err) } indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) + var manifestDeleted bool ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestRef: @@ -3886,9 +3879,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if err := json.NewEncoder(w).Encode(index_1); err != nil { - t.Errorf("failed to write response: %v", err) - } + w.Write(indexJSON_1) case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) @@ -3901,6 +3892,10 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { gotReferrerIndex = buf.Bytes() w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_1.Digest.String(): + manifestDeleted = true + // no "Docker-Content-Digest" header for manifest deletion + w.WriteHeader(http.StatusAccepted) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3923,19 +3918,22 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { } err = repo.PushReference(ctx, manifestDesc, bytes.NewReader(manifestJSON), manifestRef) if err != nil { - t.Fatalf("Manifests.Push() error = %v", err) + t.Fatalf("Manifests.PushReference() error = %v", err) } if !bytes.Equal(gotManifest, manifestJSON) { - t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + t.Errorf("Manifests.PushReference() = %v, want %v", string(gotManifest), string(manifestJSON)) } if !bytes.Equal(gotReferrerIndex, indexJSON_2) { t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } + if !manifestDeleted { + t.Errorf("manifestDeleted = %v, want %v", manifestDeleted, true) + } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test push image manifest with subject again + // test push image manifest with subject again, referrers list should not be changed ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3953,21 +3951,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+subjectDesc.Digest.String(): w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if err := json.NewEncoder(w).Encode(index_2); err != nil { - t.Errorf("failed to write response: %v", err) - } - case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: - if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { - w.WriteHeader(http.StatusBadRequest) - break - } - buf := bytes.NewBuffer(nil) - if _, err := buf.ReadFrom(r.Body); err != nil { - t.Errorf("fail to read: %v", err) - } - gotReferrerIndex = buf.Bytes() - w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) - w.WriteHeader(http.StatusCreated) + w.Write(indexJSON_2) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3995,7 +3979,7 @@ func Test_ManifestStore_PushReference_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotManifest, manifestJSON) { t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) } - // referrers list should not change + // referrers list should not be changed if !bytes.Equal(gotReferrerIndex, indexJSON_2) { t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } From a9a93d115f0041c860626aa8259d438e7e6db8e0 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 19:25:40 +0800 Subject: [PATCH 13/17] minor refactor Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 88b07c9e..b32ab6a7 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -508,8 +508,10 @@ func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Desc } referrers := filterReferrers(index.Manifests, artifactType) - if err := fn(referrers); err != nil { - return ocispec.Descriptor{}, err + if len(referrers) > 0 { + if err := fn(referrers); err != nil { + return ocispec.Descriptor{}, err + } } return desc, nil } From 22a6ca1f9198d5354ad453ff776e2e93b580386a Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 19:51:17 +0800 Subject: [PATCH 14/17] refactor pushWithIndexing Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 71 +++++++++++++++-------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index b32ab6a7..345d07ab 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -905,24 +905,7 @@ func (s *manifestStore) Fetch(ctx context.Context, target ocispec.Descriptor) (r // Push pushes the content, matching the expected descriptor. func (s *manifestStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { - switch expected.MediaType { - case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: - if state := s.repo.loadReferrersState(); state == referrersStateSupported { - // referrers API is available, no client-side indexing needed - return s.push(ctx, expected, content, expected.Digest.String()) - } - - // buffer content for referrers indexing - var buf bytes.Buffer - lr := limitReader(content, s.repo.MaxMetadataBytes) - tr := io.TeeReader(lr, &buf) - if err := s.push(ctx, expected, tr, expected.Digest.String()); err != nil { - return err - } - return s.indexReferrersForPush(ctx, expected, &buf) - default: - return s.push(ctx, expected, content, expected.Digest.String()) - } + return s.pushWithIndexing(ctx, expected, content, expected.Digest.String()) } // Exists returns true if the described content exists. @@ -1036,25 +1019,7 @@ func (s *manifestStore) PushReference(ctx context.Context, expected ocispec.Desc if err != nil { return err } - - switch expected.MediaType { - case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: - if state := s.repo.loadReferrersState(); state == referrersStateSupported { - // referrers API is available, no client-side indexing needed - return s.push(ctx, expected, content, ref.Reference) - } - - // buffer content for referrers indexing - var buf bytes.Buffer - lr := limitReader(content, s.repo.MaxMetadataBytes) - tr := io.TeeReader(lr, &buf) - if err := s.push(ctx, expected, tr, ref.Reference); err != nil { - return err - } - return s.indexReferrersForPush(ctx, expected, &buf) - default: - return s.push(ctx, expected, content, ref.Reference) - } + return s.pushWithIndexing(ctx, expected, content, ref.Reference) } // push pushes the manifest content, matching the expected descriptor. @@ -1113,15 +1078,41 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } +// pushWithIndexing pushes the manifest content, and indexes referrers for it +// if needed. +func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.Descriptor, r io.Reader, reference string) error { + switch expected.MediaType { + case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: + if state := s.repo.loadReferrersState(); state == referrersStateSupported { + // referrers API is available, no client-side indexing needed + return s.push(ctx, expected, r, reference) + } + + if err := limitSize(expected, s.repo.MaxMetadataBytes); err != nil { + return err + } + manifestJSON, err := content.ReadAll(r, expected) + if err != nil { + return err + } + if err := s.push(ctx, expected, bytes.NewReader(manifestJSON), reference); err != nil { + return err + } + return s.indexReferrersForPush(ctx, expected, manifestJSON) + default: + return s.push(ctx, expected, r, reference) + } +} + // indexReferrersForPush indexes referrers for image or artifact manifest with // the subject field. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests-with-subject -func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, r io.Reader) error { +func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error { var subject ocispec.Descriptor switch desc.MediaType { case ocispec.MediaTypeArtifactManifest: var artifact ocispec.Artifact - if err := decodeJSON(r, desc, &artifact); err != nil { + if err := json.Unmarshal(manifestJSON, &artifact); err != nil { return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) } if artifact.Subject == nil { @@ -1133,7 +1124,7 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. desc.Annotations = artifact.Annotations case ocispec.MediaTypeImageManifest: var manifest ocispec.Manifest - if err := decodeJSON(r, desc, &manifest); err != nil { + if err := json.Unmarshal(manifestJSON, &manifest); err != nil { return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) } if manifest.Subject == nil { From bcffb82bdf1e95c216a2ede3c3a3220c27609a91 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 20:48:49 +0800 Subject: [PATCH 15/17] refactor Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 103 +++++++++++++++++----------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 345d07ab..3131fc67 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -388,10 +388,7 @@ func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, art if state == referrersStateUnsupported { // The repository is known to not support Referrers API, fallback to // referrers tag schema. - if _, err := r.referrersByTagSchema(ctx, desc, artifactType, fn); err != nil && !errors.Is(err, errdef.ErrNotFound) { - return err - } - return nil + return r.referrersByTagSchema(ctx, desc, artifactType, fn) } err := r.referrersByAPI(ctx, desc, artifactType, fn) @@ -406,10 +403,7 @@ func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, art // A 404 returned by Referrers API indicates that Referrers API is // not supported. Fallback to referrers tag schema. r.SetReferrersCapability(false) - if _, err := r.referrersByTagSchema(ctx, desc, artifactType, fn); err != nil && !errors.Is(err, errdef.ErrNotFound) { - return err - } - return nil + return r.referrersByTagSchema(ctx, desc, artifactType, fn) } return err } @@ -486,34 +480,45 @@ func (r *Repository) referrersPageByAPI(ctx context.Context, artifactType string return parseLink(resp) } -// referrersByTagSchema lists the descriptors of manifests directly referencing -// the given manifest descriptor by requesting referrers tag. +// referrersByTagSchema lists the descriptors of manifests directly +// referencing the given manifest descriptor by requesting referrers tag. // fn is called for the referrers result. If artifactType is not empty, // only referrers of the same artifact type are fed to fn. // reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#backwards-compatibility -func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) (ocispec.Descriptor, error) { +func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error { referrersTag := buildReferrersTag(desc) + _, referrers, err := r.referrersFromIndex(ctx, referrersTag) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // no referrers to the manifest + return nil + } + return err + } + + filtered := filterReferrers(referrers, artifactType) + if len(filtered) == 0 { + return nil + } + return fn(filtered) +} + +func (r *Repository) referrersFromIndex(ctx context.Context, referrersTag string) (ocispec.Descriptor, []ocispec.Descriptor, error) { desc, rc, err := r.FetchReference(ctx, referrersTag) if err != nil { - return ocispec.Descriptor{}, err + return ocispec.Descriptor{}, nil, err } defer rc.Close() if err := limitSize(desc, r.MaxMetadataBytes); err != nil { - return ocispec.Descriptor{}, fmt.Errorf("failed to read referrers index from referrers tag %s: %w", referrersTag, err) + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to read referrers index from referrers tag %s: %w", referrersTag, err) } var index ocispec.Index if err := decodeJSON(rc, desc, &index); err != nil { - return ocispec.Descriptor{}, fmt.Errorf("failed to decode referrers index from referrers tag %s: %w", referrersTag, err) + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to decode referrers index from referrers tag %s: %w", referrersTag, err) } - referrers := filterReferrers(index.Manifests, artifactType) - if len(referrers) > 0 { - if err := fn(referrers); err != nil { - return ocispec.Descriptor{}, err - } - } - return desc, nil + return desc, index.Manifests, nil } // buildReferrersTag builds the referrers tag for the given manifest descriptor. @@ -1098,30 +1103,30 @@ func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.D if err := s.push(ctx, expected, bytes.NewReader(manifestJSON), reference); err != nil { return err } - return s.indexReferrersForPush(ctx, expected, manifestJSON) + return s.indexReferrers(ctx, expected, manifestJSON) default: return s.push(ctx, expected, r, reference) } } -// indexReferrersForPush indexes referrers for image or artifact manifest with +// indexReferrers indexes referrers for image or artifact manifest with // the subject field. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests-with-subject -func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error { +func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error { var subject ocispec.Descriptor switch desc.MediaType { case ocispec.MediaTypeArtifactManifest: - var artifact ocispec.Artifact - if err := json.Unmarshal(manifestJSON, &artifact); err != nil { + var manifest ocispec.Artifact + if err := json.Unmarshal(manifestJSON, &manifest); err != nil { return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) } - if artifact.Subject == nil { + if manifest.Subject == nil { // no subject, no indexing needed return nil } - subject = *artifact.Subject - desc.ArtifactType = artifact.ArtifactType - desc.Annotations = artifact.Annotations + subject = *manifest.Subject + desc.ArtifactType = manifest.ArtifactType + desc.Annotations = manifest.Annotations case ocispec.MediaTypeImageManifest: var manifest ocispec.Manifest if err := json.Unmarshal(manifestJSON, &manifest); err != nil { @@ -1146,39 +1151,33 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. // referrers API is available, no client-side indexing needed return nil } + return s.updateReferrersIndex(ctx, desc, subject) +} +func (s *manifestStore) updateReferrersIndex(ctx context.Context, desc, subject ocispec.Descriptor) error { // there can be multiple go-routines updating the referrers tag concurrently referrersTag := buildReferrersTag(subject) s.repo.lockReferrersTag(referrersTag) defer s.repo.unlockReferrersTag(referrersTag) - var updatedReferrers []ocispec.Descriptor - var skipUpdate, skipDelete bool - oldIndexDesc, err := s.repo.referrersByTagSchema(ctx, subject, "", func(referrers []ocispec.Descriptor) error { - for _, r := range referrers { - if content.Equal(r, desc) { - // desc is already in the referrers list, skip update - skipUpdate = true - break - } - updatedReferrers = append(updatedReferrers, r) - } - return nil - }) + var skipDelete bool + oldIndexDesc, referrers, err := s.repo.referrersFromIndex(ctx, referrersTag) if err != nil { - if errors.Is(err, errdef.ErrNotFound) { - // there is no old referrers index, skip delete - skipDelete = true - } else { + if !errors.Is(err, errdef.ErrNotFound) { return err } - } - if skipUpdate { - return nil + // no old index found, skip delete + skipDelete = true } - updatedReferrers = append(updatedReferrers, desc) - newIndexDesc, newIndex, err := generateReferrersIndex(updatedReferrers) + for _, r := range referrers { + if content.Equal(r, desc) { + // desc is already in the referrers list, skip update + return nil + } + } + referrers = append(referrers, desc) + newIndexDesc, newIndex, err := generateReferrersIndex(referrers) if err != nil { return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) } From b2834bf059aa7adf8a186f01e947ecaac5b4123c Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 25 Oct 2022 20:53:36 +0800 Subject: [PATCH 16/17] rename function Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 13 ++++++------- registry/remote/repository_test.go | 12 ++++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 3131fc67..f6b7e425 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -1177,7 +1177,7 @@ func (s *manifestStore) updateReferrersIndex(ctx context.Context, desc, subject } } referrers = append(referrers, desc) - newIndexDesc, newIndex, err := generateReferrersIndex(referrers) + newIndexDesc, newIndex, err := generateIndex(referrers) if err != nil { return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) } @@ -1368,18 +1368,17 @@ func verifyContentDigest(resp *http.Response, expected digest.Digest) error { return nil } -// generateReferrersIndex generates an image index containing the given -// referrers list in its manifests field. -func generateReferrersIndex(referrers []ocispec.Descriptor) (ocispec.Descriptor, []byte, error) { - if referrers == nil { - referrers = []ocispec.Descriptor{} // make it an empty array to prevent potential server-side bugs +// generateIndex generates an image index containing the given manifests list. +func generateIndex(manifests []ocispec.Descriptor) (ocispec.Descriptor, []byte, error) { + if manifests == nil { + manifests = []ocispec.Descriptor{} // make it an empty array to prevent potential server-side bugs } index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version }, MediaType: ocispec.MediaTypeImageIndex, - Manifests: referrers, + Manifests: manifests, } indexJSON, err := json.Marshal(index) if err != nil { diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index 8d96b5a4..a3ded7d9 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -4744,7 +4744,7 @@ func Test_getReferrersTag(t *testing.T) { } } -func Test_generateReferrersIndex(t *testing.T) { +func Test_generateIndex(t *testing.T) { referrer_1 := ocispec.Artifact{ MediaType: ocispec.MediaTypeArtifactManifest, ArtifactType: "foo", @@ -4794,28 +4794,28 @@ func Test_generateReferrersIndex(t *testing.T) { tests := []struct { name string - referrers []ocispec.Descriptor + manifests []ocispec.Descriptor wantDesc ocispec.Descriptor wantBytes []byte wantErr bool }{ { name: "non-empty referrers list", - referrers: referrers, + manifests: referrers, wantDesc: wantIndexDesc, wantBytes: wantIndexJSON, wantErr: false, }, { name: "nil referrers list", - referrers: nil, + manifests: nil, wantDesc: wantEmptyIndexDesc, wantBytes: wantEmptyIndexJSON, wantErr: false, }, { name: "empty referrers list", - referrers: nil, + manifests: nil, wantDesc: wantEmptyIndexDesc, wantBytes: wantEmptyIndexJSON, wantErr: false, @@ -4823,7 +4823,7 @@ func Test_generateReferrersIndex(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1, err := generateReferrersIndex(tt.referrers) + got, got1, err := generateIndex(tt.manifests) if (err != nil) != tt.wantErr { t.Errorf("generateReferrersIndex() error = %v, wantErr %v", err, tt.wantErr) return From 95c056f6c8b598a739d487c53240b6c1c1f38cde Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Wed, 26 Oct 2022 15:27:37 +0800 Subject: [PATCH 17/17] update docs Signed-off-by: Lixia (Sylvia) Lei --- registry/remote/repository.go | 8 ++++++-- registry/remote/utils.go | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/registry/remote/repository.go b/registry/remote/repository.go index f6b7e425..1f8bf730 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -503,6 +503,9 @@ func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Desc return fn(filtered) } +// referrersFromIndex queries the referrers index using the the given referrers +// tag. If Succeeded, returns the descriptor of referrers index and the +// referrers list. func (r *Repository) referrersFromIndex(ctx context.Context, referrersTag string) (ocispec.Descriptor, []ocispec.Descriptor, error) { desc, rc, err := r.FetchReference(ctx, referrersTag) if err != nil { @@ -1083,8 +1086,8 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c return verifyContentDigest(resp, expected.Digest) } -// pushWithIndexing pushes the manifest content, and indexes referrers for it -// if needed. +// pushWithIndexing pushes the manifest content matching the expected descriptor, +// and indexes referrers for the manifest when needed. func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.Descriptor, r io.Reader, reference string) error { switch expected.MediaType { case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: @@ -1154,6 +1157,7 @@ func (s *manifestStore) indexReferrers(ctx context.Context, desc ocispec.Descrip return s.updateReferrersIndex(ctx, desc, subject) } +// updateReferrersIndex updates the referrers index for desc referencing subject. func (s *manifestStore) updateReferrersIndex(ctx context.Context, desc, subject ocispec.Descriptor) error { // there can be multiple go-routines updating the referrers tag concurrently referrersTag := buildReferrersTag(subject) diff --git a/registry/remote/utils.go b/registry/remote/utils.go index 7564cca9..797169f4 100644 --- a/registry/remote/utils.go +++ b/registry/remote/utils.go @@ -74,7 +74,11 @@ func limitSize(desc ocispec.Descriptor, n int64) error { n = defaultMaxMetadataBytes } if desc.Size > n { - return errdef.ErrSizeExceedsLimit + return fmt.Errorf( + "content size %v exceeds MaxMetadataBytes %v: %w", + desc.Size, + n, + errdef.ErrSizeExceedsLimit) } return nil }