feat: add flag to skip deleting obsolete referrers index (oras-projec…

…t#957)

Signed-off-by: Billy Zha <jinzha1@microsoft.com>
Co-authored-by: Terry Howe <tlhowe@amazon.com>

cmd/oras/internal/errors/errors.go

@@ -16,9 +16,11 @@ limitations under the License.
 package errors
 
 import (
+	"errors"
 	"fmt"
 
 	"oras.land/oras-go/v2/registry"
+	"oras.land/oras-go/v2/registry/remote"
 )
 
 // NewErrInvalidReference creates a new error based on the reference string.
@@ -30,3 +32,9 @@ func NewErrInvalidReference(ref registry.Reference) error {
 func NewErrInvalidReferenceStr(ref string) error {
 	return fmt.Errorf("%s: invalid image reference, expecting <name:tag|name@digest>", ref)
 }
+
+// IsReferrersIndexDelete checks if err is a referrers index delete error.
+func IsReferrersIndexDelete(err error) bool {
+	var re *remote.ReferrersError
+	return errors.As(err, &re) && re.IsReferrersIndexDelete()
+}

cmd/oras/internal/option/referrers.go

@@ -0,0 +1,42 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package option
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"oras.land/oras-go/v2/registry/remote"
+)
+
+// Referrers option struct.
+type Referrers struct {
+	SkipDeleteReferrers bool
+}
+
+// ApplyFlags applies flags to a command flag set.
+func (opts *Referrers) ApplyFlags(fs *pflag.FlagSet) {
+	fs.BoolVarP(&opts.SkipDeleteReferrers, "skip-delete-referrers", "", false, "skip deleting old referrers index, only work on registry when referrers API is not supported")
+}
+
+// SetReferrersGC sets the referrers GC option for the passed-in target.
+func (opts *Referrers) SetReferrersGC(target any, logger logrus.FieldLogger) {
+	if repo, ok := target.(*remote.Repository); ok {
+		repo.SkipReferrersGC = opts.SkipDeleteReferrers
+	} else if opts.SkipDeleteReferrers {
+		// not a registry, can't skip referrers deletion
+		logger.Warnln("referrers deletion can only be enforced upon registry")
+	}
+}

cmd/oras/root/attach.go

@@ -19,13 +19,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras-go/v2/content/file"
+	oerr "oras.land/oras/cmd/oras/internal/errors"
 	"oras.land/oras/cmd/oras/internal/option"
 	"oras.land/oras/internal/graph"
 )
@@ -35,6 +37,7 @@ type attachOptions struct {
 	option.Packer
 	option.ImageSpec
 	option.Target
+	option.Referrers
 
 	artifactType string
 	concurrency  int
@@ -98,7 +101,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder
 }
 
 func runAttach(ctx context.Context, opts attachOptions) error {
-	ctx, _ = opts.WithContext(ctx)
+	ctx, logger := opts.WithContext(ctx)
 	annotations, err := opts.LoadManifestAnnotations()
 	if err != nil {
 		return err
@@ -121,6 +124,8 @@ func runAttach(ctx context.Context, opts attachOptions) error {
 	if err := opts.EnsureReferenceNotEmpty(); err != nil {
 		return err
 	}
+	opts.SetReferrersGC(dst, logger)
+
 	subject, err := dst.Resolve(ctx, opts.Reference)
 	if err != nil {
 		return err
@@ -163,6 +168,9 @@ func runAttach(ctx context.Context, opts attachOptions) error {
 
 	root, err := pushArtifact(dst, pack, copy)
 	if err != nil {
+		if oerr.IsReferrersIndexDelete(err) {
+			fmt.Fprintln(os.Stderr, "attached successfully but failed to remove the outdated referrers index, please use `--skip-delete-referrers` if you want to skip the deletion")
+		}
 		return err
 	}
 

cmd/oras/root/cp.go

@@ -18,6 +18,7 @@ package root
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/spf13/cobra"
 	"oras.land/oras-go/v2"
 	"oras.land/oras/cmd/oras/internal/display"
+	oerr "oras.land/oras/cmd/oras/internal/errors"
 	"oras.land/oras/cmd/oras/internal/option"
 	"oras.land/oras/internal/graph"
 )
@@ -34,6 +36,7 @@ type copyOptions struct {
 	option.Common
 	option.Platform
 	option.BinaryTarget
+	option.Referrers
 
 	recursive   bool
 	concurrency int
@@ -96,7 +99,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned:
 }
 
 func runCopy(ctx context.Context, opts copyOptions) error {
-	ctx, _ = opts.WithContext(ctx)
+	ctx, logger := opts.WithContext(ctx)
 
 	// Prepare source
 	src, err := opts.From.NewReadonlyTarget(ctx, opts.Common)
@@ -112,6 +115,7 @@ func runCopy(ctx context.Context, opts copyOptions) error {
 	if err != nil {
 		return err
 	}
+	opts.SetReferrersGC(dst, logger)
 
 	// Prepare copy options
 	committed := &sync.Map{}
@@ -167,6 +171,9 @@ func runCopy(ctx context.Context, opts copyOptions) error {
 		}
 	}
 	if err != nil {
+		if oerr.IsReferrersIndexDelete(err) {
+			fmt.Fprintln(os.Stderr, "failed to remove the outdated referrers index, please use `--skip-delete-referrers` if you want to skip the deletion")
+		}
 		return err
 	}
 

cmd/oras/root/manifest/push.go

@@ -29,6 +29,7 @@ import (
 	"oras.land/oras-go/v2/errdef"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras/cmd/oras/internal/display"
+	oerr "oras.land/oras/cmd/oras/internal/errors"
 	"oras.land/oras/cmd/oras/internal/option"
 	"oras.land/oras/internal/file"
 )
@@ -38,6 +39,7 @@ type pushOptions struct {
 	option.Descriptor
 	option.Pretty
 	option.Target
+	option.Referrers
 
 	concurrency int
 	extraRefs   []string
@@ -104,13 +106,14 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit
 }
 
 func pushManifest(ctx context.Context, opts pushOptions) error {
-	ctx, _ = opts.WithContext(ctx)
+	ctx, logger := opts.WithContext(ctx)
 	var target oras.Target
 	var err error
 	target, err = opts.NewTarget(opts.Common)
 	if err != nil {
 		return err
 	}
+	opts.SetReferrersGC(target, logger)
 	if repo, ok := target.(*remote.Repository); ok {
 		target = repo.Manifests()
 	}
@@ -151,6 +154,9 @@ func pushManifest(ctx context.Context, opts pushOptions) error {
 			return err
 		}
 		if _, err := oras.TagBytes(ctx, target, mediaType, contentBytes, ref); err != nil {
+			if oerr.IsReferrersIndexDelete(err) {
+				fmt.Fprintln(os.Stderr, "pushed successfully but failed to remove the outdated referrers index, please use `--skip-delete-referrers` if you want to skip the deletion")
+			}
 			return err
 		}
 		if err = display.PrintStatus(desc, "Uploaded ", verbose); err != nil {

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/term v0.9.0
 	gopkg.in/yaml.v3 v3.0.1
-	oras.land/oras-go/v2 v2.2.0
+	oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6
 )
 
 require (

go.sum

@@ -38,3 +38,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
 oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=
+oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6 h1:2P1fjq1znGLo7tjy9PJsZrFF5L+qywbv28IgzKEX62E=
+oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=

test/e2e/README.md

@@ -30,7 +30,7 @@ go test oras.land/oras/test/e2e/suite/${suite_name}
 This is super handy when you want to do step-by-step debugging from command-line or via an IDE. If you need to debug certain specs, use [focused specs](https://onsi.github.io/ginkgo/#focused-specs) but don't check it in.
 
 ### 4. Testing Registry Services
-The backend of E2E tests are two registry services: [oras-distribution](https://github.com/oras-project/distribution) and [upstream distribution](https://github.com/distribution/distribution). The former is expected to support image and artifact media types and referrer API; The latter is expected to only support image media type with subject and provide referrers via [tag schema](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#referrers-tag-schema). 
+The backend of E2E tests are two registry services: [oras-distribution](https://github.com/oras-project/distribution) and [upstream distribution](https://github.com/distribution/distribution). The former is expected to support image and artifact media types and referrer API; The latter is expected to only support image media type with subject and provide referrers via [tag schema](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#referrers-tag-schema), with deletion disabled. 
 
 You can run scenario test suite against your own registry services via setting `ORAS_REGISTRY_HOST` or `ORAS_REGISTRY_FALLBACK_HOST` environmental variables.
 

test/e2e/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc.3
 	gopkg.in/yaml.v2 v2.4.0
-	oras.land/oras-go/v2 v2.2.0
+	oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6
 )
 
 require (

test/e2e/go.sum

@@ -47,5 +47,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
-oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=
+oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6 h1:2P1fjq1znGLo7tjy9PJsZrFF5L+qywbv28IgzKEX62E=
+oras.land/oras-go/v2 v2.2.1-0.20230531090906-7dd0378382c6/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=

test/e2e/internal/testdata/foobar/const.go

@@ -16,6 +16,8 @@ limitations under the License.
 package foobar
 
 import (
+	"fmt"
+
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"oras.land/oras/test/e2e/internal/utils/match"
@@ -24,6 +26,8 @@ import (
 var (
 	Tag              = "foobar"
 	Digest           = "sha256:fd6ed2f36b5465244d5dc86cb4e7df0ab8a9d24adc57825099f522fe009a22bb"
+	Size             = 851
+	DescriptorStr    = fmt.Sprintf(`{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"%s","size":%d}`, Digest, Size)
 	ManifestStateKey = match.StateKey{Digest: "fd6ed2f36b54", Name: "application/vnd.oci.image.manifest.v1+json"}
 
 	FileLayerNames = []string{

test/e2e/scripts/common.sh

@@ -15,13 +15,14 @@
 
 help () {
     echo "Usage"
-    echo "  run-registry <mount-root> <image-name> <container-name> <container-port>"
+    echo "  run-registry <mount-root> <image-name> <container-name> <container-port> <deletion-enabled>"
     echo ""
     echo "Arguments"
     echo "  mount-root       root mounting directory for pre-baked registry storage files."
     echo "  image-name       image name of the registry."
     echo "  container-name   container name of the registry service."
     echo "  container-port   port to export the registry service."
+    echo "  delete-enabled   if set to true, the registry service will be configured to allow deletion."
 }
 
 # run registry service for testing
@@ -61,7 +62,7 @@ run_registry () {
     try_clean_up $ctr_name
     docker run --pull always -d -p $ctr_port:5000 --rm --name $ctr_name \
         -u $(id -u $(whoami)) \
-        --env REGISTRY_STORAGE_DELETE_ENABLED=true \
+        --env REGISTRY_STORAGE_DELETE_ENABLED=$5 \
         --env REGISTRY_AUTH_HTPASSWD_REALM=test-basic \
         --env REGISTRY_AUTH_HTPASSWD_PATH=/etc/docker/registry/passwd \
         --mount type=bind,source=$mnt_root/docker,target=/var/lib/registry/docker \

test/e2e/scripts/e2e.sh

@@ -49,14 +49,16 @@ run_registry \
   ${e2e_root}/testdata/distribution/mount \
   ghcr.io/oras-project/registry:v1.0.0-rc.4 \
   $oras_container_name \
-  $ORAS_REGISTRY_PORT
+  $ORAS_REGISTRY_PORT \
+  true
 
 echo " === preparing upstream distribution === "
 run_registry \
   ${e2e_root}/testdata/distribution/mount_fallback \
   registry:2.8.1 \
   $upstream_container_name \
-  $ORAS_REGISTRY_FALLBACK_PORT
+  $ORAS_REGISTRY_FALLBACK_PORT \
+  false
 
 echo " === run tests === "
 if ! ginkgo -r -p --succinct suite; then