Skip to content

Commit

Permalink
feat!: support OCI artifact manifest with descriptor migration (#334)
Browse files Browse the repository at this point in the history 
BREAKING CHANGE: change the parameter type of `Referrers`
Signed-off-by: Lixia (Sylvia) Lei <lixlei@microsoft.com>
  • Loading branch information
@Wwwsylvia
Wwwsylvia authored Sep 28, 2022
1 parent 3f9653f commit c1f42a4
Show file tree
Hide file tree
Showing 15 changed files with 195 additions and 261 deletions.
12 changes: 4 additions & 8 deletions content/file/file_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
Expand Down Expand Up @@ -2126,17 +2125,14 @@ func TestStore_Predecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
17 changes: 16 additions & 1 deletion content/graph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
return nil, err
}
return index.Manifests, nil
case artifactspec.MediaTypeArtifactManifest:
case artifactspec.MediaTypeArtifactManifest: // TODO: deprecate
content, err := FetchAll(ctx, fetcher, node)
if err != nil {
return nil, err
Expand All @@ -92,6 +92,21 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
nodes = append(nodes, descriptor.ArtifactToOCI(blob))
}
return nodes, nil
case ocispec.MediaTypeArtifactManifest:
content, err := FetchAll(ctx, fetcher, node)
if err != nil {
return nil, err
}

var manifest ocispec.Artifact
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err
}
var nodes []ocispec.Descriptor
if manifest.Subject != nil {
nodes = append(nodes, *manifest.Subject)
}
return append(nodes, manifest.Blobs...), nil
}
return nil, nil
}
13 changes: 4 additions & 9 deletions content/memory/memory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,11 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/cas"
"oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/resolver"
)

Expand Down Expand Up @@ -332,17 +330,14 @@ func TestStorePredecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
24 changes: 8 additions & 16 deletions content/oci/oci_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,12 @@ import (

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/cas"
"oras.land/oras-go/v2/internal/descriptor"
)

// storageTracker tracks storage API counts.
Expand Down Expand Up @@ -671,17 +669,14 @@ func TestStore_Predecessors(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down Expand Up @@ -786,17 +781,14 @@ func TestStore_ExistingStore(t *testing.T) {
}

generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down
39 changes: 15 additions & 24 deletions example_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/content/oci"
Expand All @@ -41,29 +40,21 @@ import (
var exampleMemoryStore oras.Target
var remoteHost string
var (
exampleManifest, _ = json.Marshal(artifactspec.Manifest{
MediaType: artifactspec.MediaTypeArtifactManifest,
exampleManifest, _ = json.Marshal(ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
ArtifactType: "example/content"})
exampleManifestDescriptor = ocispec.Descriptor{
MediaType: artifactspec.MediaTypeArtifactManifest,
MediaType: ocispec.MediaTypeArtifactManifest,
Digest: digest.Digest(digest.FromBytes(exampleManifest)),
Size: int64(len(exampleManifest))}
exampleManifestDescriptorArtifactspec = artifactspec.Descriptor{
MediaType: exampleManifestDescriptor.MediaType,
Digest: exampleManifestDescriptor.Digest,
Size: exampleManifestDescriptor.Size}
exampleSignatureManifest, _ = json.Marshal(artifactspec.Manifest{
MediaType: artifactspec.MediaTypeArtifactManifest,
exampleSignatureManifest, _ = json.Marshal(ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
ArtifactType: "example/signature",
Subject: &exampleManifestDescriptorArtifactspec})
Subject: &exampleManifestDescriptor})
exampleSignatureManifestDescriptor = ocispec.Descriptor{
MediaType: artifactspec.MediaTypeArtifactManifest,
MediaType: ocispec.MediaTypeArtifactManifest,
Digest: digest.FromBytes(exampleSignatureManifest),
Size: int64(len(exampleSignatureManifest))}
exampleSignatureManifestDescriptorArtifactspec = artifactspec.Descriptor{
MediaType: exampleSignatureManifestDescriptor.MediaType,
Digest: exampleSignatureManifestDescriptor.Digest,
Size: exampleSignatureManifestDescriptor.Size}
)

func pushBlob(ctx context.Context, mediaType string, blob []byte, target oras.Target) (desc ocispec.Descriptor, err error) {
Expand Down Expand Up @@ -126,15 +117,15 @@ func TestMain(m *testing.M) {
case strings.Contains(p, "/blobs/uploads/"+exampleUploadUUid) && m == "GET":
w.WriteHeader(http.StatusCreated)
case strings.Contains(p, "/manifests/"+string(exampleSignatureManifestDescriptor.Digest)):
w.Header().Set("Content-Type", artifactspec.MediaTypeArtifactManifest)
w.Header().Set("Content-Type", ocispec.MediaTypeArtifactManifest)
w.Header().Set("Docker-Content-Digest", string(exampleSignatureManifestDescriptor.Digest))
w.Header().Set("Content-Length", strconv.Itoa(len(exampleSignatureManifest)))
w.Write(exampleSignatureManifest)
case strings.Contains(p, "/manifests/latest") && m == "PUT":
w.WriteHeader(http.StatusCreated)
case strings.Contains(p, "/manifests/"+string(exampleManifestDescriptor.Digest)),
strings.Contains(p, "/manifests/latest") && m == "HEAD":
w.Header().Set("Content-Type", artifactspec.MediaTypeArtifactManifest)
w.Header().Set("Content-Type", ocispec.MediaTypeArtifactManifest)
w.Header().Set("Docker-Content-Digest", string(exampleManifestDescriptor.Digest))
w.Header().Set("Content-Length", strconv.Itoa(len(exampleManifest)))
if m == "GET" {
Expand All @@ -143,14 +134,14 @@ func TestMain(m *testing.M) {
case strings.Contains(p, "/artifacts/referrers"):
w.Header().Set("ORAS-Api-Version", "oras/1.0")
q := r.URL.Query()
var referrers []artifactspec.Descriptor
var referrers []ocispec.Descriptor
if q.Get("digest") == exampleManifestDescriptor.Digest.String() {
referrers = []artifactspec.Descriptor{exampleSignatureManifestDescriptorArtifactspec}
referrers = []ocispec.Descriptor{exampleSignatureManifestDescriptor}
} else if q.Get("digest") == exampleSignatureManifestDescriptor.Digest.String() {
referrers = []artifactspec.Descriptor{}
referrers = []ocispec.Descriptor{}
}
result := struct {
Referrers []artifactspec.Descriptor `json:"referrers"`
Referrers []ocispec.Descriptor `json:"referrers"`
}{
Referrers: referrers,
}
Expand Down Expand Up @@ -321,7 +312,7 @@ func Example_copyArtifactManifestRemoteToLocal() {
dst := memory.New()
ctx := context.Background()

exampleDigest := "sha256:f9308ac4616a808210c12d049b4eb684754a5acf2c3c8d353a9fa2b3c47c274a"
exampleDigest := "sha256:70c29a81e235dda5c2cebb8ec06eafd3cca346cbd91f15ac74cefd98681c5b3d"
descriptor, err := src.Resolve(ctx, exampleDigest)
if err != nil {
panic(err)
Expand Down Expand Up @@ -362,5 +353,5 @@ func Example_extendedCopyArtifactAndReferrersRemoteToLocal() {

fmt.Println(desc.Digest)
// Output:
// sha256:1f3e679d4fc05dca20a699ae5af5fb2b7d481d5694aff929165d1c8b0f4c8598
// sha256:f396bc4d300934a39ca28ab0d5ac8a3573336d7d63c654d783a68cd1e2057662
}
4 changes: 2 additions & 2 deletions extendedcopy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,11 +287,11 @@ func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) {
// findReferrersAndFilter filters the predecessors with Referrers.
func findReferrersAndFilter(rf registry.ReferrerFinder, ctx context.Context, desc ocispec.Descriptor, regex *regexp.Regexp) ([]ocispec.Descriptor, error) {
var predecessors []ocispec.Descriptor
if err := rf.Referrers(ctx, desc, "", func(referrers []artifactspec.Descriptor) error {
if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error {
// for each page of the results, do the following:
for _, referrer := range referrers {
if regex.MatchString(referrer.ArtifactType) {
predecessors = append(predecessors, descriptor.ArtifactToOCI(referrer))
predecessors = append(predecessors, referrer)
}
}
return nil
Expand Down
54 changes: 21 additions & 33 deletions extendedcopy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,17 +68,14 @@ func TestExtendedCopy_FullCopy(t *testing.T) {
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
Expand Down Expand Up @@ -169,17 +166,14 @@ func TestExtendedCopyGraph_FullCopy(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -377,17 +371,14 @@ func TestExtendedCopyGraph_WithDepthOption(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -512,17 +503,14 @@ func TestExtendedCopyGraph_WithFindPredecessorsOption(t *testing.T) {
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}
generateArtifactManifest := func(subject ocispec.Descriptor, blobs ...ocispec.Descriptor) {
var manifest artifactspec.Manifest
artifactSubject := descriptor.OCIToArtifact(subject)
manifest.Subject = &artifactSubject
for _, blob := range blobs {
manifest.Blobs = append(manifest.Blobs, descriptor.OCIToArtifact(blob))
}
var manifest ocispec.Artifact
manifest.Subject = &subject
manifest.Blobs = append(manifest.Blobs, blobs...)
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0
Expand Down Expand Up @@ -577,7 +565,7 @@ func TestExtendedCopyGraph_WithFindPredecessorsOption(t *testing.T) {
for _, p := range predecessors {
// filter media type
switch p.MediaType {
case artifactspec.MediaTypeArtifactManifest:
case ocispec.MediaTypeArtifactManifest:
filtered = append(filtered, p)
}
}
Expand Down Expand Up @@ -1054,7 +1042,7 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
var referrerSet []artifactspec.Descriptor
var referrerSet []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
Expand All @@ -1075,7 +1063,7 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
appendBlob(artifactspec.MediaTypeArtifactManifest, manifestJSON)
}
pushReferrers := func(desc ocispec.Descriptor, artifactType string) {
referrerSet = append(referrerSet, artifactspec.Descriptor{
referrerSet = append(referrerSet, ocispec.Descriptor{
MediaType: desc.MediaType,
ArtifactType: artifactType,
Digest: desc.Digest,
Expand Down Expand Up @@ -1132,12 +1120,12 @@ func TestExtendedCopyGraph_FilterArtifactTypeByReferrersWithMultipleRegex(t *tes
w.Write(blobs[5])
case strings.Contains(p, "referrers"):
q := r.URL.Query()
var referrers []artifactspec.Descriptor
var referrers []ocispec.Descriptor
if q.Get("digest") == descs[0].Digest.String() {
referrers = referrerSet
}
result := struct {
Referrers []artifactspec.Descriptor `json:"referrers"`
Referrers []ocispec.Descriptor `json:"referrers"`
}{
Referrers: referrers,
}
Expand Down
Loading

0 comments on commit c1f42a4

Please sign in to comment.