From 654ac4fa8f304bb3b2d2c1eafa8a25744bd2caac Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Thu, 11 Aug 2022 13:52:36 +0800 Subject: [PATCH 01/19] feat: support push a blob to a remote registry Signed-off-by: Zoey Li --- cmd/oras/blob/cmd.go | 62 +++++++++++++++++++++++++++ cmd/oras/blob/push.go | 98 +++++++++++++++++++++++++++++++++++++++++++ cmd/oras/file.go | 4 +- cmd/oras/main.go | 2 + 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 cmd/oras/blob/cmd.go create mode 100644 cmd/oras/blob/push.go diff --git a/cmd/oras/blob/cmd.go b/cmd/oras/blob/cmd.go new file mode 100644 index 000000000..760e605be --- /dev/null +++ b/cmd/oras/blob/cmd.go @@ -0,0 +1,62 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blob + +import ( + "github.com/spf13/cobra" + "oras.land/oras/cmd/oras/internal/option" +) + +func Cmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "blob [command]", + Short: "Blob operations", + } + + cmd.AddCommand(pushCmd()) + return cmd +} + +func pushCmd() *cobra.Command { + var opts pushBlobOptions + cmd := &cobra.Command{ + Use: "push name[:tag|@digest] file", + Short: "Push a blob to remote registry", + Long: `Push a blob to remote registry + +Example - Push blob "hi.txt": + oras blob push localhost:5000/hello:latest hi.txt + +Example - Push blob to the insecure registry: + oras blob push localhost:5000/hello:latest hi.txt --insecure + +Example - Push blob to the HTTP registry: + oras blob push localhost:5000/hello:latest hi.txt --plain-http +`, + Args: cobra.ExactArgs(2), + PreRunE: func(cmd *cobra.Command, args []string) error { + return opts.ReadPassword() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.targetRef = args[0] + opts.FileRef = args[1] + return pushBlob(opts) + }, + } + + option.ApplyFlags(&opts, cmd.Flags()) + return cmd +} diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go new file mode 100644 index 000000000..782a6e0dc --- /dev/null +++ b/cmd/oras/blob/push.go @@ -0,0 +1,98 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blob + +import ( + "context" + "fmt" + "io" + "os" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/cmd/oras/internal/option" +) + +type pushBlobOptions struct { + option.Common + option.Remote + + FileRef string + targetRef string +} + +func pushBlob(opts pushBlobOptions) error { + ctx, _ := opts.SetLoggerLevel() + repo, err := opts.NewRepository(opts.targetRef, opts.Common) + if err != nil { + return err + } + + // prepare blob content + desc, fp, err := packBlob(ctx, &opts) + if err != nil { + return err + } + defer func() { + closeErr := fp.Close() + if err == nil { + err = closeErr + } + }() + + // push blob + if err = repo.Push(ctx, desc, fp); err != nil { + return err + } + + fmt.Println("Pushed", opts.targetRef) + fmt.Println("Digest:", desc.Digest) + + return nil +} + +func packBlob(ctx context.Context, opts *pushBlobOptions) (ocispec.Descriptor, *os.File, error) { + filename := opts.FileRef + if filename == "" { + return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") + } + + fp, err := os.Open(filename) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", filename, err) + } + + fi, err := os.Stat(filename) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", filename, err) + } + + dgst, err := digest.FromReader(fp) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + + if _, err = fp.Seek(0, io.SeekStart); err != nil { + return ocispec.Descriptor{}, nil, err + } + + desc := ocispec.Descriptor{ + MediaType: "application/octet-stream", + Digest: dgst, + Size: fi.Size(), + } + return desc, fp, nil +} diff --git a/cmd/oras/file.go b/cmd/oras/file.go index ffdfeaa1a..5b5d1acb7 100644 --- a/cmd/oras/file.go +++ b/cmd/oras/file.go @@ -28,11 +28,13 @@ func loadFiles(ctx context.Context, store *file.Store, annotations map[string]ma var files []ocispec.Descriptor for _, fileRef := range fileRefs { filename, mediaType := parseFileReference(fileRef, "") + + // get shortest absolute path as unique name name := filepath.Clean(filename) if !filepath.IsAbs(name) { - // convert to slash-separated path unless it is absolute path name = filepath.ToSlash(name) } + if verbose { fmt.Println("Preparing", name) } diff --git a/cmd/oras/main.go b/cmd/oras/main.go index dd688c6fb..89d01916d 100644 --- a/cmd/oras/main.go +++ b/cmd/oras/main.go @@ -4,6 +4,7 @@ import ( "os" "github.com/spf13/cobra" + "oras.land/oras/cmd/oras/blob" "oras.land/oras/cmd/oras/manifest" "oras.land/oras/cmd/oras/repository" "oras.land/oras/cmd/oras/tag" @@ -23,6 +24,7 @@ func main() { discoverCmd(), copyCmd(), attachCmd(), + blob.Cmd(), manifest.Cmd(), tag.TagCmd(), repository.Cmd(), From 3391f71d37538a1a084bdae92629c96b6896df63 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 12 Aug 2022 11:07:55 +0800 Subject: [PATCH 02/19] [PRFix move func PrepareBlob to internal package, add exists check on the desc, fix typos] Signed-off-by: Zoey Li --- cmd/oras/blob/cmd.go | 9 +++--- cmd/oras/blob/push.go | 63 +++++++++++---------------------------- internal/upload/upload.go | 59 ++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 internal/upload/upload.go diff --git a/cmd/oras/blob/cmd.go b/cmd/oras/blob/cmd.go index 760e605be..d50063557 100644 --- a/cmd/oras/blob/cmd.go +++ b/cmd/oras/blob/cmd.go @@ -23,7 +23,7 @@ import ( func Cmd() *cobra.Command { cmd := &cobra.Command{ Use: "blob [command]", - Short: "Blob operations", + Short: "[Preview] Blob operations", } cmd.AddCommand(pushCmd()) @@ -34,8 +34,9 @@ func pushCmd() *cobra.Command { var opts pushBlobOptions cmd := &cobra.Command{ Use: "push name[:tag|@digest] file", - Short: "Push a blob to remote registry", - Long: `Push a blob to remote registry + Short: "[Preview] Push a blob to remote registry", + Long: `[Preview] Push a blob to remote registry +** This command is in preview and under development. ** Example - Push blob "hi.txt": oras blob push localhost:5000/hello:latest hi.txt @@ -52,7 +53,7 @@ Example - Push blob to the HTTP registry: }, RunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] - opts.FileRef = args[1] + opts.fileRef = args[1] return pushBlob(opts) }, } diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 782a6e0dc..a59a4c6d6 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -16,25 +16,22 @@ limitations under the License. package blob import ( - "context" "fmt" - "io" - "os" - digest "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/upload" ) type pushBlobOptions struct { option.Common option.Remote - FileRef string + fileRef string targetRef string } -func pushBlob(opts pushBlobOptions) error { +func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { @@ -42,57 +39,33 @@ func pushBlob(opts pushBlobOptions) error { } // prepare blob content - desc, fp, err := packBlob(ctx, &opts) + desc, fp, err := upload.PrepareContent(opts.fileRef, "application/octet-stream") if err != nil { return err } defer func() { - closeErr := fp.Close() - if err == nil { + if closeErr := fp.Close(); err == nil { err = closeErr } }() - // push blob - if err = repo.Push(ctx, desc, fp); err != nil { + exists, err := repo.Exists(ctx, desc) + if err != nil { return err } + if exists { + statusPrinter := display.StatusPrinter("Exists ", opts.Verbose) + if err := statusPrinter(ctx, desc); err != nil { + return err + } + } else { + if err = repo.Push(ctx, desc, fp); err != nil { + return err + } + } fmt.Println("Pushed", opts.targetRef) fmt.Println("Digest:", desc.Digest) return nil } - -func packBlob(ctx context.Context, opts *pushBlobOptions) (ocispec.Descriptor, *os.File, error) { - filename := opts.FileRef - if filename == "" { - return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") - } - - fp, err := os.Open(filename) - if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", filename, err) - } - - fi, err := os.Stat(filename) - if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", filename, err) - } - - dgst, err := digest.FromReader(fp) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - - if _, err = fp.Seek(0, io.SeekStart); err != nil { - return ocispec.Descriptor{}, nil, err - } - - desc := ocispec.Descriptor{ - MediaType: "application/octet-stream", - Digest: dgst, - Size: fi.Size(), - } - return desc, fp, nil -} diff --git a/internal/upload/upload.go b/internal/upload/upload.go new file mode 100644 index 000000000..603f18ab6 --- /dev/null +++ b/internal/upload/upload.go @@ -0,0 +1,59 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upload + +import ( + "fmt" + "io" + "os" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// PrepareContent opens the given file and generates descriptor from the given +// file. +func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { + if path == "" { + return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") + } + + fp, err := os.Open(path) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) + } + + fi, err := os.Stat(path) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) + } + + dgst, err := digest.FromReader(fp) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + + if _, err = fp.Seek(0, io.SeekStart); err != nil { + return ocispec.Descriptor{}, nil, err + } + + desc := ocispec.Descriptor{ + MediaType: mediaType, + Digest: dgst, + Size: fi.Size(), + } + return desc, fp, nil +} From c38346953a54b445bcffd3ffe0f444c5970b5b7a Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 12 Aug 2022 18:30:45 +0800 Subject: [PATCH 03/19] [PRFix add test] Signed-off-by: Zoey Li --- cmd/oras/blob/cmd.go | 8 +-- cmd/oras/blob/push.go | 13 ++-- internal/{upload/upload.go => file/file.go} | 5 +- internal/file/file_test.go | 75 +++++++++++++++++++++ 4 files changed, 89 insertions(+), 12 deletions(-) rename internal/{upload/upload.go => file/file.go} (93%) create mode 100644 internal/file/file_test.go diff --git a/cmd/oras/blob/cmd.go b/cmd/oras/blob/cmd.go index d50063557..0eac07a40 100644 --- a/cmd/oras/blob/cmd.go +++ b/cmd/oras/blob/cmd.go @@ -33,19 +33,19 @@ func Cmd() *cobra.Command { func pushCmd() *cobra.Command { var opts pushBlobOptions cmd := &cobra.Command{ - Use: "push name[:tag|@digest] file", + Use: "push name[@digest] file [flags]", Short: "[Preview] Push a blob to remote registry", Long: `[Preview] Push a blob to remote registry ** This command is in preview and under development. ** Example - Push blob "hi.txt": - oras blob push localhost:5000/hello:latest hi.txt + oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt Example - Push blob to the insecure registry: - oras blob push localhost:5000/hello:latest hi.txt --insecure + oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt --insecure Example - Push blob to the HTTP registry: - oras blob push localhost:5000/hello:latest hi.txt --plain-http + oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt --plain-http `, Args: cobra.ExactArgs(2), PreRunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index a59a4c6d6..8b26a85cf 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -17,10 +17,11 @@ package blob import ( "fmt" + "strings" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/upload" + "oras.land/oras/internal/file" ) type pushBlobOptions struct { @@ -39,13 +40,15 @@ func pushBlob(opts pushBlobOptions) (err error) { } // prepare blob content - desc, fp, err := upload.PrepareContent(opts.fileRef, "application/octet-stream") + desc, rc, err := file.PrepareContent(opts.fileRef, "application/octet-stream") if err != nil { return err } defer func() { - if closeErr := fp.Close(); err == nil { - err = closeErr + if closeErr := rc.Close(); err == nil { + if closeErr != nil && !strings.Contains(closeErr.Error(), "file already closed") { + err = closeErr + } } }() @@ -59,7 +62,7 @@ func pushBlob(opts pushBlobOptions) (err error) { return err } } else { - if err = repo.Push(ctx, desc, fp); err != nil { + if err = repo.Push(ctx, desc, rc); err != nil { return err } } diff --git a/internal/upload/upload.go b/internal/file/file.go similarity index 93% rename from internal/upload/upload.go rename to internal/file/file.go index 603f18ab6..040c41041 100644 --- a/internal/upload/upload.go +++ b/internal/file/file.go @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package upload +package file import ( "fmt" @@ -24,8 +24,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// PrepareContent opens the given file and generates descriptor from the given -// file. +// PrepareContent prepares the content descriptor from the file. func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") diff --git a/internal/file/file_test.go b/internal/file/file_test.go new file mode 100644 index 000000000..a2e2f0444 --- /dev/null +++ b/internal/file/file_test.go @@ -0,0 +1,75 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package file_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/internal/file" +) + +func TestFile_PrepareContent(t *testing.T) { + // generate test content + tempDir := t.TempDir() + dirPath := filepath.Join(tempDir, "testdir") + if err := os.MkdirAll(dirPath, 0777); err != nil { + t.Fatal("error calling Mkdir(), error =", err) + } + content := []byte("hello world!") + fileName := "test.txt" + path := filepath.Join(dirPath, fileName) + if err := ioutil.WriteFile(path, content, 0444); err != nil { + t.Fatal("error calling WriteFile(), error =", err) + } + + blobMediaType := "application/octet-stream" + wantDesc := ocispec.Descriptor{ + MediaType: blobMediaType, + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + + // test PrepareContent + gotDesc, rc, err := file.PrepareContent(path, blobMediaType) + defer rc.Close() + if err != nil { + t.Fatal("PrepareContent() error=", err) + } + if !reflect.DeepEqual(gotDesc, wantDesc) { + t.Errorf("PrepareContent() = %v, want %v", gotDesc, wantDesc) + } + + // test PrepareContent with missing file name + _, _, err = file.PrepareContent("", blobMediaType) + expected := "missing file name" + if err.Error() != expected { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) + } + + // test PrepareContent with nonexistent file + _, _, err = file.PrepareContent("nonexistent.txt", blobMediaType) + expected = "failed to open nonexistent.txt" + if !strings.Contains(err.Error(), expected) { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) + } +} From e13e539cb3f66ba1e534ef7de0a647ace9b0f729 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Mon, 15 Aug 2022 14:01:38 +0800 Subject: [PATCH 04/19] [PRFix modify command msg] Signed-off-by: Zoey Li --- cmd/oras/blob/cmd.go | 37 +++--------------------------- cmd/oras/blob/push.go | 39 ++++++++++++++++++++++++------- internal/file/file_test.go | 47 ++++++++++++++++++++++++++------------ 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/cmd/oras/blob/cmd.go b/cmd/oras/blob/cmd.go index 0eac07a40..9c7f3f1ce 100644 --- a/cmd/oras/blob/cmd.go +++ b/cmd/oras/blob/cmd.go @@ -17,7 +17,6 @@ package blob import ( "github.com/spf13/cobra" - "oras.land/oras/cmd/oras/internal/option" ) func Cmd() *cobra.Command { @@ -26,38 +25,8 @@ func Cmd() *cobra.Command { Short: "[Preview] Blob operations", } - cmd.AddCommand(pushCmd()) - return cmd -} - -func pushCmd() *cobra.Command { - var opts pushBlobOptions - cmd := &cobra.Command{ - Use: "push name[@digest] file [flags]", - Short: "[Preview] Push a blob to remote registry", - Long: `[Preview] Push a blob to remote registry -** This command is in preview and under development. ** - -Example - Push blob "hi.txt": - oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt - -Example - Push blob to the insecure registry: - oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt --insecure - -Example - Push blob to the HTTP registry: - oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt --plain-http -`, - Args: cobra.ExactArgs(2), - PreRunE: func(cmd *cobra.Command, args []string) error { - return opts.ReadPassword() - }, - RunE: func(cmd *cobra.Command, args []string) error { - opts.targetRef = args[0] - opts.fileRef = args[1] - return pushBlob(opts) - }, - } - - option.ApplyFlags(&opts, cmd.Flags()) + cmd.AddCommand( + pushCmd(), + ) return cmd } diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 8b26a85cf..be36af389 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -17,8 +17,8 @@ package blob import ( "fmt" - "strings" + "github.com/spf13/cobra" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" "oras.land/oras/internal/file" @@ -32,6 +32,35 @@ type pushBlobOptions struct { targetRef string } +func pushCmd() *cobra.Command { + var opts pushBlobOptions + cmd := &cobra.Command{ + Use: "push file [flags]", + Short: "[Preview] Push a blob to remote registry", + Long: `[Preview] Push a blob to remote registry +** This command is in preview and under development. ** + +Example - Push blob "hi.txt": + oras blob push localhost:5000/hello hi.txt + +Example - Push blob to the insecure registry: + oras blob push localhost:5000/hello hi.txt --insecure +`, + Args: cobra.ExactArgs(2), + PreRunE: func(cmd *cobra.Command, args []string) error { + return opts.ReadPassword() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.targetRef = args[0] + opts.fileRef = args[1] + return pushBlob(opts) + }, + } + + option.ApplyFlags(&opts, cmd.Flags()) + return cmd +} + func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() repo, err := opts.NewRepository(opts.targetRef, opts.Common) @@ -44,13 +73,7 @@ func pushBlob(opts pushBlobOptions) (err error) { if err != nil { return err } - defer func() { - if closeErr := rc.Close(); err == nil { - if closeErr != nil && !strings.Contains(closeErr.Error(), "file already closed") { - err = closeErr - } - } - }() + defer rc.Close() exists, err := repo.Exists(ctx, desc) if err != nil { diff --git a/internal/file/file_test.go b/internal/file/file_test.go index a2e2f0444..f6ef878f3 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -16,7 +16,6 @@ limitations under the License. package file_test import ( - "io/ioutil" "os" "path/filepath" "reflect" @@ -31,44 +30,64 @@ import ( func TestFile_PrepareContent(t *testing.T) { // generate test content tempDir := t.TempDir() - dirPath := filepath.Join(tempDir, "testdir") - if err := os.MkdirAll(dirPath, 0777); err != nil { - t.Fatal("error calling Mkdir(), error =", err) - } content := []byte("hello world!") fileName := "test.txt" - path := filepath.Join(dirPath, fileName) - if err := ioutil.WriteFile(path, content, 0444); err != nil { + path := filepath.Join(tempDir, fileName) + if err := os.WriteFile(path, content, 0444); err != nil { t.Fatal("error calling WriteFile(), error =", err) } blobMediaType := "application/octet-stream" - wantDesc := ocispec.Descriptor{ + want := ocispec.Descriptor{ MediaType: blobMediaType, Digest: digest.FromBytes(content), Size: int64(len(content)), } // test PrepareContent - gotDesc, rc, err := file.PrepareContent(path, blobMediaType) + got, rc, err := file.PrepareContent(path, blobMediaType) defer rc.Close() if err != nil { t.Fatal("PrepareContent() error=", err) } - if !reflect.DeepEqual(gotDesc, wantDesc) { - t.Errorf("PrepareContent() = %v, want %v", gotDesc, wantDesc) + if !reflect.DeepEqual(got, want) { + t.Errorf("PrepareContent() = %v, want %v", got, want) + } +} + +func TestFile_PrepareContent_errMissingFileName(t *testing.T) { + // generate test content + tempDir := t.TempDir() + content := []byte("hello world!") + fileName := "test.txt" + path := filepath.Join(tempDir, fileName) + if err := os.WriteFile(path, content, 0444); err != nil { + t.Fatal("error calling WriteFile(), error =", err) } + blobMediaType := "application/octet-stream" // test PrepareContent with missing file name - _, _, err = file.PrepareContent("", blobMediaType) + _, _, err := file.PrepareContent("", blobMediaType) expected := "missing file name" if err.Error() != expected { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) } +} + +func TestFile_PrepareContent_errOpenFile(t *testing.T) { + // generate test content + tempDir := t.TempDir() + content := []byte("hello world!") + fileName := "test.txt" + path := filepath.Join(tempDir, fileName) + if err := os.WriteFile(path, content, 0444); err != nil { + t.Fatal("error calling WriteFile(), error =", err) + } + blobMediaType := "application/octet-stream" // test PrepareContent with nonexistent file - _, _, err = file.PrepareContent("nonexistent.txt", blobMediaType) - expected = "failed to open nonexistent.txt" + _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType) + expected := "failed to open nonexistent.txt" if !strings.Contains(err.Error(), expected) { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) } From 366ba0fbe66aa904b93776fd14e1911400cb6791 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Wed, 24 Aug 2022 13:46:46 +0800 Subject: [PATCH 05/19] [PRFix modify test and use display.PrintStatus] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 5 ++--- internal/file/file.go | 5 ++--- internal/file/file_test.go | 31 ++++++++++--------------------- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index be36af389..186be4867 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -43,7 +43,7 @@ func pushCmd() *cobra.Command { Example - Push blob "hi.txt": oras blob push localhost:5000/hello hi.txt -Example - Push blob to the insecure registry: +Example - Push blob without TLS: oras blob push localhost:5000/hello hi.txt --insecure `, Args: cobra.ExactArgs(2), @@ -80,8 +80,7 @@ func pushBlob(opts pushBlobOptions) (err error) { return err } if exists { - statusPrinter := display.StatusPrinter("Exists ", opts.Verbose) - if err := statusPrinter(ctx, desc); err != nil { + if err := display.PrintStatus(desc, "Exists ", opts.Verbose); err != nil { return err } } else { diff --git a/internal/file/file.go b/internal/file/file.go index 040c41041..ab9d36c57 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -49,10 +49,9 @@ func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadC return ocispec.Descriptor{}, nil, err } - desc := ocispec.Descriptor{ + return ocispec.Descriptor{ MediaType: mediaType, Digest: dgst, Size: fi.Size(), - } - return desc, fp, nil + }, fp, nil } diff --git a/internal/file/file_test.go b/internal/file/file_test.go index f6ef878f3..201108349 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -16,6 +16,7 @@ limitations under the License. package file_test import ( + "io" "os" "path/filepath" "reflect" @@ -27,6 +28,8 @@ import ( "oras.land/oras/internal/file" ) +const blobMediaType = "application/mock-octet-stream" + func TestFile_PrepareContent(t *testing.T) { // generate test content tempDir := t.TempDir() @@ -37,7 +40,6 @@ func TestFile_PrepareContent(t *testing.T) { t.Fatal("error calling WriteFile(), error =", err) } - blobMediaType := "application/octet-stream" want := ocispec.Descriptor{ MediaType: blobMediaType, Digest: digest.FromBytes(content), @@ -53,19 +55,16 @@ func TestFile_PrepareContent(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Errorf("PrepareContent() = %v, want %v", got, want) } + actualContent, err := io.ReadAll(rc) + if err != nil { + t.Fatal("PrepareContent(): not able to read content from rc, error=", err) + } + if !reflect.DeepEqual(actualContent, content) { + t.Errorf("PrepareContent() = %v, want %v", actualContent, content) + } } func TestFile_PrepareContent_errMissingFileName(t *testing.T) { - // generate test content - tempDir := t.TempDir() - content := []byte("hello world!") - fileName := "test.txt" - path := filepath.Join(tempDir, fileName) - if err := os.WriteFile(path, content, 0444); err != nil { - t.Fatal("error calling WriteFile(), error =", err) - } - blobMediaType := "application/octet-stream" - // test PrepareContent with missing file name _, _, err := file.PrepareContent("", blobMediaType) expected := "missing file name" @@ -75,16 +74,6 @@ func TestFile_PrepareContent_errMissingFileName(t *testing.T) { } func TestFile_PrepareContent_errOpenFile(t *testing.T) { - // generate test content - tempDir := t.TempDir() - content := []byte("hello world!") - fileName := "test.txt" - path := filepath.Join(tempDir, fileName) - if err := os.WriteFile(path, content, 0444); err != nil { - t.Fatal("error calling WriteFile(), error =", err) - } - blobMediaType := "application/octet-stream" - // test PrepareContent with nonexistent file _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType) expected := "failed to open nonexistent.txt" From 7e1326c892124ce2bfd6d0e5b5d6181379e31621 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Wed, 24 Aug 2022 20:28:14 +0800 Subject: [PATCH 06/19] [PRFix modify print message] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 186be4867..3667a25e5 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -36,8 +36,8 @@ func pushCmd() *cobra.Command { var opts pushBlobOptions cmd := &cobra.Command{ Use: "push file [flags]", - Short: "[Preview] Push a blob to remote registry", - Long: `[Preview] Push a blob to remote registry + Short: "[Preview] Push a blob to a remote registry", + Long: `[Preview] Push a blob to a remote registry ** This command is in preview and under development. ** Example - Push blob "hi.txt": @@ -80,7 +80,7 @@ func pushBlob(opts pushBlobOptions) (err error) { return err } if exists { - if err := display.PrintStatus(desc, "Exists ", opts.Verbose); err != nil { + if err := display.PrintStatus(desc, "Exists", opts.Verbose); err != nil { return err } } else { From 2063c0de8dbc1ef7d03ac9f36c780c6da9e605b0 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Thu, 25 Aug 2022 21:38:22 +0800 Subject: [PATCH 07/19] [PRFix add new flags] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 29 +++++++++++++- cmd/oras/internal/option/output.go | 52 +++++++++++++++++++++++++ cmd/oras/internal/option/output_test.go | 29 ++++++++++++++ internal/file/file.go | 17 +++++++- 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 cmd/oras/internal/option/output.go create mode 100644 cmd/oras/internal/option/output_test.go diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 3667a25e5..f11428a94 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -16,6 +16,8 @@ limitations under the License. package blob import ( + "encoding/json" + "errors" "fmt" "github.com/spf13/cobra" @@ -26,6 +28,7 @@ import ( type pushBlobOptions struct { option.Common + option.Output option.Remote fileRef string @@ -43,6 +46,15 @@ func pushCmd() *cobra.Command { Example - Push blob "hi.txt": oras blob push localhost:5000/hello hi.txt +Example - Push blob from stdin: +oras blob push localhost:5000/hello - + +Example - Push blob "hi.txt" and output the descriptor + oras blob push localhost:5000/hello hi.txt --descriptor + +Example - Push blob "hi.txt" and output the prettified descriptor + oras blob push localhost:5000/hello hi.txt --descriptor --pretty + Example - Push blob without TLS: oras blob push localhost:5000/hello hi.txt --insecure `, @@ -63,6 +75,11 @@ Example - Push blob without TLS: func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() + + if opts.fileRef == "-" && opts.PasswordFromStdin { + return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + } + repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err @@ -80,7 +97,8 @@ func pushBlob(opts pushBlobOptions) (err error) { return err } if exists { - if err := display.PrintStatus(desc, "Exists", opts.Verbose); err != nil { + verbose := opts.Verbose && !opts.OutputDescriptor + if err := display.PrintStatus(desc, "Exists", verbose); err != nil { return err } } else { @@ -89,6 +107,15 @@ func pushBlob(opts pushBlobOptions) (err error) { } } + if opts.OutputDescriptor { + descBytes, err := json.Marshal(desc) + if err != nil { + return fmt.Errorf("failed to marshal blob: %w", err) + } + err = opts.OutputContent(descBytes) + return err + } + fmt.Println("Pushed", opts.targetRef) fmt.Println("Digest:", desc.Digest) diff --git a/cmd/oras/internal/option/output.go b/cmd/oras/internal/option/output.go new file mode 100644 index 000000000..a314fc9b0 --- /dev/null +++ b/cmd/oras/internal/option/output.go @@ -0,0 +1,52 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package option + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/spf13/pflag" +) + +// Output option struct. +type Output struct { + OutputDescriptor bool + Pretty bool +} + +// ApplyFlags applies flags to a command flag set. +func (opts *Output) ApplyFlags(fs *pflag.FlagSet) { + fs.BoolVarP(&opts.OutputDescriptor, "descriptor", "", false, "output descriptor") + fs.BoolVarP(&opts.Pretty, "pretty", "", false, "output prettified content") +} + +// OutputContent outputs the content to stdout +func (opts *Output) OutputContent(content []byte) error { + if opts.Pretty { + buf := bytes.NewBuffer(nil) + if err := json.Indent(buf, content, "", " "); err != nil { + return fmt.Errorf("failed to prettify: %w", err) + } + buf.WriteByte('\n') + content = buf.Bytes() + } + + _, err := os.Stdout.Write(content) + return err +} diff --git a/cmd/oras/internal/option/output_test.go b/cmd/oras/internal/option/output_test.go new file mode 100644 index 000000000..e3770cb3c --- /dev/null +++ b/cmd/oras/internal/option/output_test.go @@ -0,0 +1,29 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package option + +import ( + "testing" + + "github.com/spf13/pflag" +) + +func TestOutput_FlagInit(t *testing.T) { + var test struct { + Output + } + ApplyFlags(&test, pflag.NewFlagSet("oras-test", pflag.ExitOnError)) +} diff --git a/internal/file/file.go b/internal/file/file.go index ab9d36c57..50b9c6a0d 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -16,6 +16,7 @@ limitations under the License. package file import ( + "bytes" "fmt" "io" "os" @@ -24,12 +25,26 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// PrepareContent prepares the content descriptor from the file. +// PrepareContent prepares the content descriptor from the file path. func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") } + if path == "-" { + content, err := io.ReadAll(os.Stdin) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + rc := io.NopCloser(bytes.NewReader(content)) + + return ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(content), + Size: int64(len(content)), + }, rc, nil + } + fp, err := os.Open(path) if err != nil { return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) From 5c636925b230ef46036dcc9aab14cb68142b04a1 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 26 Aug 2022 20:58:58 +0800 Subject: [PATCH 08/19] =?UTF-8?q?=EF=BB=BF[PRFix=20rebase]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 24 ++++++------ cmd/oras/internal/option/output.go | 52 ------------------------- cmd/oras/internal/option/output_test.go | 29 -------------- 3 files changed, 13 insertions(+), 92 deletions(-) delete mode 100644 cmd/oras/internal/option/output.go delete mode 100644 cmd/oras/internal/option/output_test.go diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index f11428a94..ec1ce9bd4 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -16,9 +16,9 @@ limitations under the License. package blob import ( - "encoding/json" "errors" "fmt" + "os" "github.com/spf13/cobra" "oras.land/oras/cmd/oras/internal/display" @@ -28,7 +28,8 @@ import ( type pushBlobOptions struct { option.Common - option.Output + option.Descriptor + option.Pretty option.Remote fileRef string @@ -96,26 +97,27 @@ func pushBlob(opts pushBlobOptions) (err error) { if err != nil { return err } - if exists { - verbose := opts.Verbose && !opts.OutputDescriptor - if err := display.PrintStatus(desc, "Exists", verbose); err != nil { - return err - } - } else { + if !exists { if err = repo.Push(ctx, desc, rc); err != nil { return err } } + // outputs blob's descriptor if opts.OutputDescriptor { - descBytes, err := json.Marshal(desc) + bytes, err := opts.Marshal(desc) if err != nil { - return fmt.Errorf("failed to marshal blob: %w", err) + return err } - err = opts.OutputContent(descBytes) + err = opts.Output(os.Stdout, bytes) return err } + if exists { + if err := display.PrintStatus(desc, "Exists", opts.Verbose); err != nil { + return err + } + } fmt.Println("Pushed", opts.targetRef) fmt.Println("Digest:", desc.Digest) diff --git a/cmd/oras/internal/option/output.go b/cmd/oras/internal/option/output.go deleted file mode 100644 index a314fc9b0..000000000 --- a/cmd/oras/internal/option/output.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package option - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - - "github.com/spf13/pflag" -) - -// Output option struct. -type Output struct { - OutputDescriptor bool - Pretty bool -} - -// ApplyFlags applies flags to a command flag set. -func (opts *Output) ApplyFlags(fs *pflag.FlagSet) { - fs.BoolVarP(&opts.OutputDescriptor, "descriptor", "", false, "output descriptor") - fs.BoolVarP(&opts.Pretty, "pretty", "", false, "output prettified content") -} - -// OutputContent outputs the content to stdout -func (opts *Output) OutputContent(content []byte) error { - if opts.Pretty { - buf := bytes.NewBuffer(nil) - if err := json.Indent(buf, content, "", " "); err != nil { - return fmt.Errorf("failed to prettify: %w", err) - } - buf.WriteByte('\n') - content = buf.Bytes() - } - - _, err := os.Stdout.Write(content) - return err -} diff --git a/cmd/oras/internal/option/output_test.go b/cmd/oras/internal/option/output_test.go deleted file mode 100644 index e3770cb3c..000000000 --- a/cmd/oras/internal/option/output_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package option - -import ( - "testing" - - "github.com/spf13/pflag" -) - -func TestOutput_FlagInit(t *testing.T) { - var test struct { - Output - } - ApplyFlags(&test, pflag.NewFlagSet("oras-test", pflag.ExitOnError)) -} From 4f5d0a8a38c8863ee3a892a712ce27843accb493 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Mon, 29 Aug 2022 15:15:45 +0800 Subject: [PATCH 09/19] [PRFix add ut] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 14 +++++------- internal/file/file.go | 2 +- internal/file/file_test.go | 45 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index ec1ce9bd4..dc5a1d8d9 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -61,11 +61,14 @@ Example - Push blob without TLS: `, Args: cobra.ExactArgs(2), PreRunE: func(cmd *cobra.Command, args []string) error { + opts.targetRef = args[0] + opts.fileRef = args[1] + if opts.fileRef == "-" && opts.PasswordFromStdin { + return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + } return opts.ReadPassword() }, RunE: func(cmd *cobra.Command, args []string) error { - opts.targetRef = args[0] - opts.fileRef = args[1] return pushBlob(opts) }, } @@ -77,10 +80,6 @@ Example - Push blob without TLS: func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() - if opts.fileRef == "-" && opts.PasswordFromStdin { - return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") - } - repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err @@ -109,8 +108,7 @@ func pushBlob(opts pushBlobOptions) (err error) { if err != nil { return err } - err = opts.Output(os.Stdout, bytes) - return err + return opts.Output(os.Stdout, bytes) } if exists { diff --git a/internal/file/file.go b/internal/file/file.go index 50b9c6a0d..6cfdc4a47 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -25,7 +25,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// PrepareContent prepares the content descriptor from the file path. +// PrepareContent prepares the content descriptor from the file path or stdin. func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 201108349..32c16118b 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -64,6 +64,51 @@ func TestFile_PrepareContent(t *testing.T) { } } +func TestFile_PrepareContent_readFromStdin(t *testing.T) { + // generate test content + content := []byte("hello world!") + tmpfile, err := os.CreateTemp("", "test") + if err != nil { + t.Fatal("error calling CreateTemp(), error =", err) + } + + defer os.Remove(tmpfile.Name()) // clean up + + if _, err := tmpfile.Write(content); err != nil { + t.Fatal("error calling Write(), error =", err) + } + if _, err := tmpfile.Seek(0, 0); err != nil { + t.Fatal("error calling Seek(), error =", err) + } + + oldStdin := os.Stdin + defer func() { os.Stdin = oldStdin }() // Restore original Stdin + + os.Stdin = tmpfile + want := ocispec.Descriptor{ + MediaType: blobMediaType, + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + + // test PrepareContent + got, rc, err := file.PrepareContent("-", blobMediaType) + defer rc.Close() + if err != nil { + t.Fatal("PrepareContent() error=", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("PrepareContent() = %v, want %v", got, want) + } + actualContent, err := io.ReadAll(rc) + if err != nil { + t.Fatal("PrepareContent(): not able to read content from rc, error=", err) + } + if !reflect.DeepEqual(actualContent, content) { + t.Errorf("PrepareContent() = %v, want %v", actualContent, content) + } +} + func TestFile_PrepareContent_errMissingFileName(t *testing.T) { // test PrepareContent with missing file name _, _, err := file.PrepareContent("", blobMediaType) From af09a198821d74c5c724200d6a3cc4117ad91721 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Tue, 30 Aug 2022 13:34:42 +0800 Subject: [PATCH 10/19] =?UTF-8?q?=EF=BB=BF[PRFix=20rebase]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 14 +++++++----- internal/file/file.go | 2 +- internal/file/file_test.go | 45 -------------------------------------- 3 files changed, 9 insertions(+), 52 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index dc5a1d8d9..ec1ce9bd4 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -61,14 +61,11 @@ Example - Push blob without TLS: `, Args: cobra.ExactArgs(2), PreRunE: func(cmd *cobra.Command, args []string) error { - opts.targetRef = args[0] - opts.fileRef = args[1] - if opts.fileRef == "-" && opts.PasswordFromStdin { - return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") - } return opts.ReadPassword() }, RunE: func(cmd *cobra.Command, args []string) error { + opts.targetRef = args[0] + opts.fileRef = args[1] return pushBlob(opts) }, } @@ -80,6 +77,10 @@ Example - Push blob without TLS: func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() + if opts.fileRef == "-" && opts.PasswordFromStdin { + return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + } + repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err @@ -108,7 +109,8 @@ func pushBlob(opts pushBlobOptions) (err error) { if err != nil { return err } - return opts.Output(os.Stdout, bytes) + err = opts.Output(os.Stdout, bytes) + return err } if exists { diff --git a/internal/file/file.go b/internal/file/file.go index 6cfdc4a47..50b9c6a0d 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -25,7 +25,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// PrepareContent prepares the content descriptor from the file path or stdin. +// PrepareContent prepares the content descriptor from the file path. func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 32c16118b..201108349 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -64,51 +64,6 @@ func TestFile_PrepareContent(t *testing.T) { } } -func TestFile_PrepareContent_readFromStdin(t *testing.T) { - // generate test content - content := []byte("hello world!") - tmpfile, err := os.CreateTemp("", "test") - if err != nil { - t.Fatal("error calling CreateTemp(), error =", err) - } - - defer os.Remove(tmpfile.Name()) // clean up - - if _, err := tmpfile.Write(content); err != nil { - t.Fatal("error calling Write(), error =", err) - } - if _, err := tmpfile.Seek(0, 0); err != nil { - t.Fatal("error calling Seek(), error =", err) - } - - oldStdin := os.Stdin - defer func() { os.Stdin = oldStdin }() // Restore original Stdin - - os.Stdin = tmpfile - want := ocispec.Descriptor{ - MediaType: blobMediaType, - Digest: digest.FromBytes(content), - Size: int64(len(content)), - } - - // test PrepareContent - got, rc, err := file.PrepareContent("-", blobMediaType) - defer rc.Close() - if err != nil { - t.Fatal("PrepareContent() error=", err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("PrepareContent() = %v, want %v", got, want) - } - actualContent, err := io.ReadAll(rc) - if err != nil { - t.Fatal("PrepareContent(): not able to read content from rc, error=", err) - } - if !reflect.DeepEqual(actualContent, content) { - t.Errorf("PrepareContent() = %v, want %v", actualContent, content) - } -} - func TestFile_PrepareContent_errMissingFileName(t *testing.T) { // test PrepareContent with missing file name _, _, err := file.PrepareContent("", blobMediaType) From 9bf3990ee2fc76d3b5644193e26edd124c0e52ed Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 2 Sep 2022 10:18:12 +0800 Subject: [PATCH 11/19] [PRFix fix comments] Signed-off-by: Zoey Li --- internal/file/file.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/file/file.go b/internal/file/file.go index 50b9c6a0d..0d225d8c8 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -50,12 +50,12 @@ func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadC return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) } - fi, err := os.Stat(path) + dgst, err := digest.FromReader(fp) if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) + return ocispec.Descriptor{}, nil, err } - dgst, err := digest.FromReader(fp) + size, err := fp.Seek(0, io.SeekCurrent) if err != nil { return ocispec.Descriptor{}, nil, err } @@ -67,6 +67,6 @@ func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadC return ocispec.Descriptor{ MediaType: mediaType, Digest: dgst, - Size: fi.Size(), + Size: size, }, fp, nil } From 479aa6fbd4e1ad4c65c3a5a2c4e271100e9d7f88 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Mon, 5 Sep 2022 00:08:38 +0800 Subject: [PATCH 12/19] [PRFix add an empty line] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index ec1ce9bd4..31c2e0252 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -42,6 +42,7 @@ func pushCmd() *cobra.Command { Use: "push file [flags]", Short: "[Preview] Push a blob to a remote registry", Long: `[Preview] Push a blob to a remote registry + ** This command is in preview and under development. ** Example - Push blob "hi.txt": From d1606bd45893d8ad9796c86b1593158adac50213 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Tue, 6 Sep 2022 15:17:45 +0800 Subject: [PATCH 13/19] =?UTF-8?q?=EF=BB=BF[PRFix=20add=20--size=20and=20--?= =?UTF-8?q?mediatype=20flags,=20support=20push=20with=20digest]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 62 ++++++++++++++++++---------- internal/file/file.go | 52 +++++++++++------------- internal/file/file_test.go | 82 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 52 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 31c2e0252..9961ce9a4 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -20,6 +20,8 @@ import ( "fmt" "os" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" @@ -33,13 +35,15 @@ type pushBlobOptions struct { option.Remote fileRef string + mediaType string + size int64 targetRef string } func pushCmd() *cobra.Command { var opts pushBlobOptions cmd := &cobra.Command{ - Use: "push file [flags]", + Use: "push name[@digest] file [flags]", Short: "[Preview] Push a blob to a remote registry", Long: `[Preview] Push a blob to a remote registry @@ -48,13 +52,19 @@ func pushCmd() *cobra.Command { Example - Push blob "hi.txt": oras blob push localhost:5000/hello hi.txt -Example - Push blob from stdin: -oras blob push localhost:5000/hello - +Example - Push blob "hi.txt" with the specific digest: + oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt -Example - Push blob "hi.txt" and output the descriptor +Example - Push blob from stdin with blob size: + oras blob push localhost:5000/hello - --size 12 + +Example - Push blob "hi.txt" and output the descriptor: oras blob push localhost:5000/hello hi.txt --descriptor -Example - Push blob "hi.txt" and output the prettified descriptor +Example - Push blob "hi.txt" with the specific returned media type in the descriptor: + oras blob push localhost:5000/hello hi.txt --media-type application/vnd.oci.image.config.v1+json --descriptor + +Example - Push blob "hi.txt" and output the prettified descriptor: oras blob push localhost:5000/hello hi.txt --descriptor --pretty Example - Push blob without TLS: @@ -62,15 +72,24 @@ Example - Push blob without TLS: `, Args: cobra.ExactArgs(2), PreRunE: func(cmd *cobra.Command, args []string) error { + opts.targetRef = args[0] + opts.fileRef = args[1] + if opts.fileRef == "-" && opts.PasswordFromStdin { + return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + } + if opts.fileRef == "-" && opts.size == 0 { + return errors.New("`--size` must be provided if the blob is read from stdin") + } + return opts.ReadPassword() }, RunE: func(cmd *cobra.Command, args []string) error { - opts.targetRef = args[0] - opts.fileRef = args[1] return pushBlob(opts) }, } + cmd.Flags().Int64VarP(&opts.size, "size", "", 0, "provide the blob size") + cmd.Flags().StringVarP(&opts.mediaType, "media-type", "", ocispec.MediaTypeImageLayer, "specify the returned media type in the descriptor if `--descriptor` is used") option.ApplyFlags(&opts, cmd.Flags()) return cmd } @@ -78,17 +97,14 @@ Example - Push blob without TLS: func pushBlob(opts pushBlobOptions) (err error) { ctx, _ := opts.SetLoggerLevel() - if opts.fileRef == "-" && opts.PasswordFromStdin { - return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") - } - repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err } // prepare blob content - desc, rc, err := file.PrepareContent(opts.fileRef, "application/octet-stream") + refDigest := digest.Digest(repo.Reference.Reference) + desc, rc, err := file.PrepareContent(opts.fileRef, "application/octet-stream", refDigest, opts.size) if err != nil { return err } @@ -98,27 +114,33 @@ func pushBlob(opts pushBlobOptions) (err error) { if err != nil { return err } - if !exists { + verbose := opts.Verbose && !opts.OutputDescriptor + if exists { + if err := display.PrintStatus(desc, "Exists", verbose); err != nil { + return err + } + } else { + if err := display.PrintStatus(desc, "Uploading", verbose); err != nil { + return err + } if err = repo.Push(ctx, desc, rc); err != nil { return err } + if err := display.PrintStatus(desc, "Uploaded ", verbose); err != nil { + return err + } } // outputs blob's descriptor if opts.OutputDescriptor { + desc.MediaType = opts.mediaType bytes, err := opts.Marshal(desc) if err != nil { return err } - err = opts.Output(os.Stdout, bytes) - return err + return opts.Output(os.Stdout, bytes) } - if exists { - if err := display.PrintStatus(desc, "Exists", opts.Verbose); err != nil { - return err - } - } fmt.Println("Pushed", opts.targetRef) fmt.Println("Digest:", desc.Digest) diff --git a/internal/file/file.go b/internal/file/file.go index 0d225d8c8..08ca28575 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -16,7 +16,6 @@ limitations under the License. package file import ( - "bytes" "fmt" "io" "os" @@ -25,48 +24,45 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// PrepareContent prepares the content descriptor from the file path. -func PrepareContent(path string, mediaType string) (ocispec.Descriptor, io.ReadCloser, error) { +// PrepareContent prepares the content descriptor from the file path or stdin. +// Use the input digest and size if they are provided. +func PrepareContent(path string, mediaType string, dgst digest.Digest, size int64) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") } + var file *os.File + var err error if path == "-" { - content, err := io.ReadAll(os.Stdin) + file = os.Stdin + } else { + file, err = os.Open(path) if err != nil { - return ocispec.Descriptor{}, nil, err + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) } - rc := io.NopCloser(bytes.NewReader(content)) - - return ocispec.Descriptor{ - MediaType: mediaType, - Digest: digest.FromBytes(content), - Size: int64(len(content)), - }, rc, nil - } - - fp, err := os.Open(path) - if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) } - dgst, err := digest.FromReader(fp) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - - size, err := fp.Seek(0, io.SeekCurrent) - if err != nil { - return ocispec.Descriptor{}, nil, err + if dgst == "" { + dgst, err = digest.FromReader(file) + if err != nil { + return ocispec.Descriptor{}, nil, err + } + if _, err = file.Seek(0, io.SeekStart); err != nil { + return ocispec.Descriptor{}, nil, err + } } - if _, err = fp.Seek(0, io.SeekStart); err != nil { - return ocispec.Descriptor{}, nil, err + if size <= 0 { + fi, err := file.Stat() + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) + } + size = fi.Size() } return ocispec.Descriptor{ MediaType: mediaType, Digest: dgst, Size: size, - }, fp, nil + }, file, nil } diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 201108349..990ad90e3 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -47,8 +47,7 @@ func TestFile_PrepareContent(t *testing.T) { } // test PrepareContent - got, rc, err := file.PrepareContent(path, blobMediaType) - defer rc.Close() + got, rc, err := file.PrepareContent(path, blobMediaType, "", 0) if err != nil { t.Fatal("PrepareContent() error=", err) } @@ -59,14 +58,89 @@ func TestFile_PrepareContent(t *testing.T) { if err != nil { t.Fatal("PrepareContent(): not able to read content from rc, error=", err) } + err = rc.Close() + if err != nil { + t.Fatal("error calling rc.Close(), error =", err) + } + if !reflect.DeepEqual(actualContent, content) { + t.Errorf("PrepareContent() = %v, want %v", actualContent, content) + } + + // test PrepareContent with provided digest and size + dgst := digest.Digest("sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5") + size := int64(12) + got, rc, err = file.PrepareContent(path, blobMediaType, dgst, size) + if err != nil { + t.Fatal("PrepareContent() error=", err) + } + want = ocispec.Descriptor{ + MediaType: blobMediaType, + Digest: dgst, + Size: size, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("PrepareContent() = %v, want %v", got, want) + } + actualContent, err = io.ReadAll(rc) + if err != nil { + t.Fatal("PrepareContent(): not able to read content from rc, error=", err) + } + err = rc.Close() + if err != nil { + t.Fatal("error calling rc.Close(), error =", err) + } if !reflect.DeepEqual(actualContent, content) { t.Errorf("PrepareContent() = %v, want %v", actualContent, content) } } +func TestFile_PrepareContent_fromStdin(t *testing.T) { + // generate test content + content := []byte("hello world!") + tmpfile, err := os.CreateTemp("", "test") + if err != nil { + t.Fatal("error calling CreateTemp(), error =", err) + } + + defer os.Remove(tmpfile.Name()) // clean up + defer tmpfile.Close() + + if _, err := tmpfile.Write(content); err != nil { + t.Fatal("error calling Write(), error =", err) + } + if _, err := tmpfile.Seek(0, 0); err != nil { + t.Fatal("error calling Seek(), error =", err) + } + + defer func(stdin *os.File) { os.Stdin = stdin }(os.Stdin) + + os.Stdin = tmpfile + wantDesc := ocispec.Descriptor{ + MediaType: blobMediaType, + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + + // test PrepareContent + gotDesc, gotRc, err := file.PrepareContent("-", blobMediaType, "", 0) + defer gotRc.Close() + if err != nil { + t.Fatal("PrepareContent() error=", err) + } + if !reflect.DeepEqual(gotDesc, wantDesc) { + t.Errorf("PrepareContent() = %v, want %v", gotDesc, wantDesc) + } + if _, err = tmpfile.Seek(0, io.SeekStart); err != nil { + t.Fatal("error calling Seek(), error =", err) + } + if !reflect.DeepEqual(gotRc, tmpfile) { + t.Errorf("PrepareContent() = %v, want %v", gotRc, tmpfile) + } +} + func TestFile_PrepareContent_errMissingFileName(t *testing.T) { // test PrepareContent with missing file name - _, _, err := file.PrepareContent("", blobMediaType) + _, _, err := file.PrepareContent("", blobMediaType, "", 0) expected := "missing file name" if err.Error() != expected { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) @@ -75,7 +149,7 @@ func TestFile_PrepareContent_errMissingFileName(t *testing.T) { func TestFile_PrepareContent_errOpenFile(t *testing.T) { // test PrepareContent with nonexistent file - _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType) + _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType, "", 0) expected := "failed to open nonexistent.txt" if !strings.Contains(err.Error(), expected) { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) From b146c68448a95af171c1dad025e5a51074f42282 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Thu, 8 Sep 2022 23:23:35 +0800 Subject: [PATCH 14/19] [PRFix fix comments] Signed-off-by: Zoey Li --- cmd/oras/blob/cmd.go | 2 +- cmd/oras/blob/fetch.go | 3 +-- cmd/oras/blob/push.go | 16 ++++++++-------- internal/file/file_test.go | 9 +++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/oras/blob/cmd.go b/cmd/oras/blob/cmd.go index a7fdddd19..85193f7df 100644 --- a/cmd/oras/blob/cmd.go +++ b/cmd/oras/blob/cmd.go @@ -27,7 +27,7 @@ func Cmd() *cobra.Command { cmd.AddCommand( fetchCmd(), - pushCmd(), + pushCmd(), ) return cmd } diff --git a/cmd/oras/blob/fetch.go b/cmd/oras/blob/fetch.go index 8c8808da5..ecff2d3e8 100644 --- a/cmd/oras/blob/fetch.go +++ b/cmd/oras/blob/fetch.go @@ -16,7 +16,6 @@ limitations under the License. package blob import ( - "encoding/json" "errors" "fmt" "io" @@ -150,7 +149,7 @@ func fetchBlob(opts fetchBlobOptions) (fetchErr error) { // outputs blob's descriptor if `--descriptor` is used if opts.OutputDescriptor { - descJSON, err := json.Marshal(desc) + descJSON, err := opts.Marshal(desc) if err != nil { return err } diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 9961ce9a4..b9959d9ab 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -43,7 +43,7 @@ type pushBlobOptions struct { func pushCmd() *cobra.Command { var opts pushBlobOptions cmd := &cobra.Command{ - Use: "push name[@digest] file [flags]", + Use: "push [flags] name[@digest] file", Short: "[Preview] Push a blob to a remote registry", Long: `[Preview] Push a blob to a remote registry @@ -56,19 +56,19 @@ Example - Push blob "hi.txt" with the specific digest: oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt Example - Push blob from stdin with blob size: - oras blob push localhost:5000/hello - --size 12 + oras blob push --size 12 localhost:5000/hello - Example - Push blob "hi.txt" and output the descriptor: - oras blob push localhost:5000/hello hi.txt --descriptor + oras blob push --descriptor localhost:5000/hello hi.txt Example - Push blob "hi.txt" with the specific returned media type in the descriptor: - oras blob push localhost:5000/hello hi.txt --media-type application/vnd.oci.image.config.v1+json --descriptor + oras blob push --media-type application/vnd.oci.image.config.v1+json --descriptor localhost:5000/hello hi.txt Example - Push blob "hi.txt" and output the prettified descriptor: - oras blob push localhost:5000/hello hi.txt --descriptor --pretty + oras blob push --descriptor --pretty localhost:5000/hello hi.txt Example - Push blob without TLS: - oras blob push localhost:5000/hello hi.txt --insecure + oras blob push --insecure localhost:5000/hello hi.txt `, Args: cobra.ExactArgs(2), PreRunE: func(cmd *cobra.Command, args []string) error { @@ -134,11 +134,11 @@ func pushBlob(opts pushBlobOptions) (err error) { // outputs blob's descriptor if opts.OutputDescriptor { desc.MediaType = opts.mediaType - bytes, err := opts.Marshal(desc) + descJSON, err := opts.Marshal(desc) if err != nil { return err } - return opts.Output(os.Stdout, bytes) + return opts.Output(os.Stdout, descJSON) } fmt.Println("Pushed", opts.targetRef) diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 990ad90e3..8d817a305 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -97,12 +97,13 @@ func TestFile_PrepareContent(t *testing.T) { func TestFile_PrepareContent_fromStdin(t *testing.T) { // generate test content content := []byte("hello world!") - tmpfile, err := os.CreateTemp("", "test") + tempDir := t.TempDir() + fileName := "test.txt" + path := filepath.Join(tempDir, fileName) + tmpfile, err := os.Create(path) if err != nil { - t.Fatal("error calling CreateTemp(), error =", err) + t.Fatal("error calling os.Create(), error =", err) } - - defer os.Remove(tmpfile.Name()) // clean up defer tmpfile.Close() if _, err := tmpfile.Write(content); err != nil { From 734dc788b5281b1e4a87c2021a113bc3c8c5b73d Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 9 Sep 2022 16:28:35 +0800 Subject: [PATCH 15/19] [PRFix fix comments] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 12 ++++-------- internal/file/file.go | 31 ++++++++++++++++++++++--------- internal/file/file_test.go | 21 +++++++++++++++------ 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index b9959d9ab..777ca112c 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -55,8 +55,8 @@ Example - Push blob "hi.txt": Example - Push blob "hi.txt" with the specific digest: oras blob push localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 hi.txt -Example - Push blob from stdin with blob size: - oras blob push --size 12 localhost:5000/hello - +Example - Push blob from stdin with blob size and digest: + oras blob push --size 12 localhost:5000/hello@sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5 - Example - Push blob "hi.txt" and output the descriptor: oras blob push --descriptor localhost:5000/hello hi.txt @@ -77,9 +77,6 @@ Example - Push blob without TLS: if opts.fileRef == "-" && opts.PasswordFromStdin { return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") } - if opts.fileRef == "-" && opts.size == 0 { - return errors.New("`--size` must be provided if the blob is read from stdin") - } return opts.ReadPassword() }, @@ -88,7 +85,7 @@ Example - Push blob without TLS: }, } - cmd.Flags().Int64VarP(&opts.size, "size", "", 0, "provide the blob size") + cmd.Flags().Int64VarP(&opts.size, "size", "", -1, "provide the blob size") cmd.Flags().StringVarP(&opts.mediaType, "media-type", "", ocispec.MediaTypeImageLayer, "specify the returned media type in the descriptor if `--descriptor` is used") option.ApplyFlags(&opts, cmd.Flags()) return cmd @@ -104,7 +101,7 @@ func pushBlob(opts pushBlobOptions) (err error) { // prepare blob content refDigest := digest.Digest(repo.Reference.Reference) - desc, rc, err := file.PrepareContent(opts.fileRef, "application/octet-stream", refDigest, opts.size) + desc, rc, err := file.PrepareContent(opts.fileRef, opts.mediaType, refDigest, opts.size) if err != nil { return err } @@ -133,7 +130,6 @@ func pushBlob(opts pushBlobOptions) (err error) { // outputs blob's descriptor if opts.OutputDescriptor { - desc.MediaType = opts.mediaType descJSON, err := opts.Marshal(desc) if err != nil { return err diff --git a/internal/file/file.go b/internal/file/file.go index 08ca28575..51f440434 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -16,6 +16,7 @@ limitations under the License. package file import ( + "errors" "fmt" "io" "os" @@ -28,19 +29,31 @@ import ( // Use the input digest and size if they are provided. func PrepareContent(path string, mediaType string, dgst digest.Digest, size int64) (ocispec.Descriptor, io.ReadCloser, error) { if path == "" { - return ocispec.Descriptor{}, nil, fmt.Errorf("missing file name") + return ocispec.Descriptor{}, nil, errors.New("missing file name") } - var file *os.File - var err error + // prepares the content descriptor from stdin if path == "-" { - file = os.Stdin - } else { - file, err = os.Open(path) - if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) + // throw err if size or digest is not provided. + if size < 0 || dgst == "" { + return ocispec.Descriptor{}, nil, errors.New("content size and digest must be provided if it is read from stdin") } + return ocispec.Descriptor{ + MediaType: mediaType, + Digest: dgst, + Size: size, + }, os.Stdin, nil + } + + file, err := os.Open(path) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) } + defer func() { + if err != nil { + file.Close() + } + }() if dgst == "" { dgst, err = digest.FromReader(file) @@ -52,7 +65,7 @@ func PrepareContent(path string, mediaType string, dgst digest.Digest, size int6 } } - if size <= 0 { + if size < 0 { fi, err := file.Stat() if err != nil { return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 8d817a305..a7c7f0588 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -47,7 +47,7 @@ func TestFile_PrepareContent(t *testing.T) { } // test PrepareContent - got, rc, err := file.PrepareContent(path, blobMediaType, "", 0) + got, rc, err := file.PrepareContent(path, blobMediaType, "", -1) if err != nil { t.Fatal("PrepareContent() error=", err) } @@ -75,7 +75,7 @@ func TestFile_PrepareContent(t *testing.T) { } want = ocispec.Descriptor{ MediaType: blobMediaType, - Digest: dgst, + Digest: digest.Digest(dgst), Size: size, } if !reflect.DeepEqual(got, want) { @@ -116,14 +116,16 @@ func TestFile_PrepareContent_fromStdin(t *testing.T) { defer func(stdin *os.File) { os.Stdin = stdin }(os.Stdin) os.Stdin = tmpfile + dgst := digest.FromBytes(content) + size := int64(len(content)) wantDesc := ocispec.Descriptor{ MediaType: blobMediaType, Digest: digest.FromBytes(content), Size: int64(len(content)), } - // test PrepareContent - gotDesc, gotRc, err := file.PrepareContent("-", blobMediaType, "", 0) + // test PrepareContent with provided digest and size + gotDesc, gotRc, err := file.PrepareContent("-", blobMediaType, dgst, size) defer gotRc.Close() if err != nil { t.Fatal("PrepareContent() error=", err) @@ -137,11 +139,18 @@ func TestFile_PrepareContent_fromStdin(t *testing.T) { if !reflect.DeepEqual(gotRc, tmpfile) { t.Errorf("PrepareContent() = %v, want %v", gotRc, tmpfile) } + + // test PrepareContent from stdin with missing digest and size + _, _, err = file.PrepareContent("-", blobMediaType, "", -1) + expected := "content size and digest must be provided if it is read from stdin" + if err.Error() != expected { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) + } } func TestFile_PrepareContent_errMissingFileName(t *testing.T) { // test PrepareContent with missing file name - _, _, err := file.PrepareContent("", blobMediaType, "", 0) + _, _, err := file.PrepareContent("", blobMediaType, "", -1) expected := "missing file name" if err.Error() != expected { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) @@ -150,7 +159,7 @@ func TestFile_PrepareContent_errMissingFileName(t *testing.T) { func TestFile_PrepareContent_errOpenFile(t *testing.T) { // test PrepareContent with nonexistent file - _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType, "", 0) + _, _, err := file.PrepareContent("nonexistent.txt", blobMediaType, "", -1) expected := "failed to open nonexistent.txt" if !strings.Contains(err.Error(), expected) { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) From 5de8d5aebec391a8618de952cb42984c7402d5b9 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 9 Sep 2022 17:39:58 +0800 Subject: [PATCH 16/19] [PRFix add digest check and ut] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 4 +--- internal/file/file.go | 17 ++++++++++++++--- internal/file/file_test.go | 18 ++++++++++++++---- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index 777ca112c..fc00b7307 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -20,7 +20,6 @@ import ( "fmt" "os" - digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras/cmd/oras/internal/display" @@ -100,8 +99,7 @@ func pushBlob(opts pushBlobOptions) (err error) { } // prepare blob content - refDigest := digest.Digest(repo.Reference.Reference) - desc, rc, err := file.PrepareContent(opts.fileRef, opts.mediaType, refDigest, opts.size) + desc, rc, err := file.PrepareContent(opts.fileRef, opts.mediaType, repo.Reference.Reference, opts.size) if err != nil { return err } diff --git a/internal/file/file.go b/internal/file/file.go index 51f440434..fc63133d1 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -26,12 +26,23 @@ import ( ) // PrepareContent prepares the content descriptor from the file path or stdin. -// Use the input digest and size if they are provided. -func PrepareContent(path string, mediaType string, dgst digest.Digest, size int64) (ocispec.Descriptor, io.ReadCloser, error) { +// Use the input digest and size if they are provided. Will return error if the +// content is from stdin but the content digest and size are missing. +func PrepareContent(path string, mediaType string, dgstStr string, size int64) (_ ocispec.Descriptor, _ io.ReadCloser, prepareErr error) { if path == "" { return ocispec.Descriptor{}, nil, errors.New("missing file name") } + // validate digest + var dgst digest.Digest + if dgstStr != "" { + var err error + dgst, err = digest.Parse(dgstStr) + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("invalid digest %s: %w", dgstStr, err) + } + } + // prepares the content descriptor from stdin if path == "-" { // throw err if size or digest is not provided. @@ -50,7 +61,7 @@ func PrepareContent(path string, mediaType string, dgst digest.Digest, size int6 return ocispec.Descriptor{}, nil, fmt.Errorf("failed to open %s: %w", path, err) } defer func() { - if err != nil { + if prepareErr != nil { file.Close() } }() diff --git a/internal/file/file_test.go b/internal/file/file_test.go index a7c7f0588..336f869da 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -16,6 +16,7 @@ limitations under the License. package file_test import ( + "errors" "io" "os" "path/filepath" @@ -67,15 +68,15 @@ func TestFile_PrepareContent(t *testing.T) { } // test PrepareContent with provided digest and size - dgst := digest.Digest("sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5") + dgstStr := "sha256:9a201d228ebd966211f7d1131be19f152be428bd373a92071c71d8deaf83b3e5" size := int64(12) - got, rc, err = file.PrepareContent(path, blobMediaType, dgst, size) + got, rc, err = file.PrepareContent(path, blobMediaType, dgstStr, size) if err != nil { t.Fatal("PrepareContent() error=", err) } want = ocispec.Descriptor{ MediaType: blobMediaType, - Digest: digest.Digest(dgst), + Digest: digest.Digest(dgstStr), Size: size, } if !reflect.DeepEqual(got, want) { @@ -125,7 +126,7 @@ func TestFile_PrepareContent_fromStdin(t *testing.T) { } // test PrepareContent with provided digest and size - gotDesc, gotRc, err := file.PrepareContent("-", blobMediaType, dgst, size) + gotDesc, gotRc, err := file.PrepareContent("-", blobMediaType, string(dgst), size) defer gotRc.Close() if err != nil { t.Fatal("PrepareContent() error=", err) @@ -148,6 +149,15 @@ func TestFile_PrepareContent_fromStdin(t *testing.T) { } } +func TestFile_PrepareContent_errDigestInvalidFormat(t *testing.T) { + // test PrepareContent from stdin with invalid digest + invalidDgst := "xyz" + _, _, err := file.PrepareContent("-", blobMediaType, invalidDgst, 12) + if !errors.Is(err, digest.ErrDigestInvalidFormat) { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, digest.ErrDigestInvalidFormat) + } +} + func TestFile_PrepareContent_errMissingFileName(t *testing.T) { // test PrepareContent with missing file name _, _, err := file.PrepareContent("", blobMediaType, "", -1) From e37222eb7018869c867a7de54d328f4c50f4cb61 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Fri, 9 Sep 2022 23:05:32 +0800 Subject: [PATCH 17/19] [PRFix validate input size if provided] Signed-off-by: Zoey Li --- internal/file/file.go | 21 +++++++++++---------- internal/file/file_test.go | 9 +++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/internal/file/file.go b/internal/file/file.go index fc63133d1..2044a8e5c 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -28,7 +28,7 @@ import ( // PrepareContent prepares the content descriptor from the file path or stdin. // Use the input digest and size if they are provided. Will return error if the // content is from stdin but the content digest and size are missing. -func PrepareContent(path string, mediaType string, dgstStr string, size int64) (_ ocispec.Descriptor, _ io.ReadCloser, prepareErr error) { +func PrepareContent(path string, mediaType string, dgstStr string, size int64) (desc ocispec.Descriptor, rc io.ReadCloser, prepareErr error) { if path == "" { return ocispec.Descriptor{}, nil, errors.New("missing file name") } @@ -66,6 +66,15 @@ func PrepareContent(path string, mediaType string, dgstStr string, size int64) ( } }() + fi, err := file.Stat() + if err != nil { + return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) + } + actualSize := fi.Size() + if size >= 0 && size != actualSize { + return ocispec.Descriptor{}, nil, fmt.Errorf("input size %d does not match the actual content size %d", size, actualSize) + } + if dgst == "" { dgst, err = digest.FromReader(file) if err != nil { @@ -76,17 +85,9 @@ func PrepareContent(path string, mediaType string, dgstStr string, size int64) ( } } - if size < 0 { - fi, err := file.Stat() - if err != nil { - return ocispec.Descriptor{}, nil, fmt.Errorf("failed to stat %s: %w", path, err) - } - size = fi.Size() - } - return ocispec.Descriptor{ MediaType: mediaType, Digest: dgst, - Size: size, + Size: actualSize, }, file, nil } diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 336f869da..d099a61a3 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -17,6 +17,7 @@ package file_test import ( "errors" + "fmt" "io" "os" "path/filepath" @@ -93,6 +94,14 @@ func TestFile_PrepareContent(t *testing.T) { if !reflect.DeepEqual(actualContent, content) { t.Errorf("PrepareContent() = %v, want %v", actualContent, content) } + + // test PrepareContent with provided size, but the size does not match the + // actual content size + _, _, err = file.PrepareContent(path, blobMediaType, "", 15) + expected := fmt.Sprintf("input size %d does not match the actual content size %d", 15, size) + if err.Error() != expected { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) + } } func TestFile_PrepareContent_fromStdin(t *testing.T) { From dc00d7053a82edc5a44287403ae919a3541a5fc0 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Tue, 13 Sep 2022 14:34:11 +0800 Subject: [PATCH 18/19] [PRFix add size check in prerun] Signed-off-by: Zoey Li --- cmd/oras/blob/push.go | 10 +++++++--- internal/file/file.go | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cmd/oras/blob/push.go b/cmd/oras/blob/push.go index fc00b7307..e79d064b5 100644 --- a/cmd/oras/blob/push.go +++ b/cmd/oras/blob/push.go @@ -73,10 +73,14 @@ Example - Push blob without TLS: PreRunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] opts.fileRef = args[1] - if opts.fileRef == "-" && opts.PasswordFromStdin { - return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + if opts.fileRef == "-" { + if opts.PasswordFromStdin { + return errors.New("`-` read file from input and `--password-stdin` read password from input cannot be both used") + } + if opts.size < 0 { + return errors.New("`--size` must be provided if the blob is read from stdin") + } } - return opts.ReadPassword() }, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/internal/file/file.go b/internal/file/file.go index 2044a8e5c..987f2c478 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -46,8 +46,11 @@ func PrepareContent(path string, mediaType string, dgstStr string, size int64) ( // prepares the content descriptor from stdin if path == "-" { // throw err if size or digest is not provided. - if size < 0 || dgst == "" { - return ocispec.Descriptor{}, nil, errors.New("content size and digest must be provided if it is read from stdin") + if size < 0 { + return ocispec.Descriptor{}, nil, errors.New("content size must be provided if it is read from stdin") + } + if dgst == "" { + return ocispec.Descriptor{}, nil, errors.New("content digest must be provided if it is read from stdin") } return ocispec.Descriptor{ MediaType: mediaType, From c8b5c29c6b7f5b245f27e71dcb6cbf9a91e06754 Mon Sep 17 00:00:00 2001 From: Zoey Li Date: Tue, 13 Sep 2022 15:02:55 +0800 Subject: [PATCH 19/19] [PRFix fix test] Signed-off-by: Zoey Li --- internal/file/file_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/file/file_test.go b/internal/file/file_test.go index d099a61a3..32e060c82 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -150,9 +150,16 @@ func TestFile_PrepareContent_fromStdin(t *testing.T) { t.Errorf("PrepareContent() = %v, want %v", gotRc, tmpfile) } - // test PrepareContent from stdin with missing digest and size + // test PrepareContent from stdin with missing size _, _, err = file.PrepareContent("-", blobMediaType, "", -1) - expected := "content size and digest must be provided if it is read from stdin" + expected := "content size must be provided if it is read from stdin" + if err.Error() != expected { + t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) + } + + // test PrepareContent from stdin with missing digest + _, _, err = file.PrepareContent("-", blobMediaType, "", 5) + expected = "content digest must be provided if it is read from stdin" if err.Error() != expected { t.Fatalf("PrepareContent() error = %v, wantErr %v", err, expected) }