From 21f4b652b3dfd500058f314c68c13cafd57d0735 Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Thu, 4 Jan 2024 13:12:13 -0600 Subject: [PATCH] Update to latest bufbuild/registry protos (#2697) Update to support the latest changes to bufbuild/registry. Migrate the bufcasalpha package to use the bufcas interface types directly (since the storagev1beta1 types are gone). --- go.mod | 1 - go.sum | 2 - make/buf/all.mk | 6 +- .../buf/bufsync/bufsyncapi/sync_handler.go | 6 +- .../buf/bufsync/bufsynctest/bufsynctest.go | 12 +- private/buf/bufsync/syncer_test.go | 3 +- private/buf/cmd/buf/command/push/push.go | 6 +- private/buf/cmd/buf/command/push/push_test.go | 20 +- private/bufpkg/bufapimodule/module_reader.go | 6 +- .../bufpkg/bufapimodule/module_reader_test.go | 13 +- .../bufpkg/bufcas/bufcasalpha/bufcasalpha.go | 122 +++--- private/bufpkg/bufcas/digest.go | 41 +- private/bufpkg/bufcas/manifest.go | 7 + private/bufpkg/bufcas/parse_error.go | 3 - private/bufpkg/bufcas/proto.go | 371 ------------------ private/pkg/syserror/syserror.go | 92 +++++ private/pkg/syserror/usage.gen.go | 19 + 17 files changed, 236 insertions(+), 494 deletions(-) delete mode 100644 private/bufpkg/bufcas/proto.go create mode 100644 private/pkg/syserror/syserror.go create mode 100644 private/pkg/syserror/usage.gen.go diff --git a/go.mod b/go.mod index b1947d9c58..31838b9dec 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.19 require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1 - buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1 connectrpc.com/connect v1.14.0 connectrpc.com/otelconnect v0.6.0 github.com/bufbuild/protocompile v0.7.1 diff --git a/go.sum b/go.sum index d4eafe8ca6..9bf8c55ead 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1 h1:u0olL4yf2p7Tl5jfsAK5keaFi+JFJuv1CDHrbiXkxkk= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1/go.mod h1:tiTMKD8j6Pd/D2WzREoweufjzaJKHZg35f/VGcZ2v3I= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1 h1:IOqyoSoI4xrCGUc1PBdr7rbDUMEYX7fondG+fdDDvMo= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1/go.mod h1:L+pboYag9Pd0vt+ErAHa7QSdqP0Dzd7S+4OLKlrXNXQ= connectrpc.com/connect v1.14.0 h1:PDS+J7uoz5Oui2VEOMcfz6Qft7opQM9hPiKvtGC01pA= connectrpc.com/connect v1.14.0/go.mod h1:uoAq5bmhhn43TwhaKdGKN/bZcGtzPW1v+ngDTn5u+8s= connectrpc.com/otelconnect v0.6.0 h1:VJAdQL9+sgdUw9+7+J+jq8pQo/h1S7tSFv2+vDcR7bU= diff --git a/make/buf/all.mk b/make/buf/all.mk index 0eafa7e98d..2df625d034 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -1,9 +1,5 @@ GO_ALL_REPO_PKGS := ./cmd/... ./private/... -# Keep bufbuild/registry pinned at version prior to landing the refactor branch. -# This allows main to continue to build until it is ready to incorporate changes from the refactor branch. -BUFBUILD_REGISTRY_VERSION := 20231205222057-ac336d436f46 -GO_GET_PKGS := $(GO_GET_PKGS) \ - buf.build/gen/go/bufbuild/registry/protocolbuffers/go@v1.32.0-$(BUFBUILD_REGISTRY_VERSION).1 +#GO_GET_PKGS := $(GO_GET_PKGS) GO_BINS := $(GO_BINS) \ cmd/buf \ cmd/protoc-gen-buf-breaking \ diff --git a/private/buf/bufsync/bufsyncapi/sync_handler.go b/private/buf/bufsync/bufsyncapi/sync_handler.go index cd9415fc96..0e8d7b94e3 100644 --- a/private/buf/bufsync/bufsyncapi/sync_handler.go +++ b/private/buf/bufsync/bufsyncapi/sync_handler.go @@ -447,15 +447,15 @@ func (h *syncHandler) syncCommitModule( if err != nil { return nil, err } - protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet) + protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet) if err != nil { return nil, err } resp, err := service.SyncGitCommit(ctx, connect.NewRequest(®istryv1alpha1.SyncGitCommitRequest{ Owner: moduleIdentity.Owner(), Repository: moduleIdentity.Repository(), - Manifest: bufcasalpha.BlobToAlpha(protoManifestBlob), - Blobs: bufcasalpha.BlobsToAlpha(protoBlobs), + Manifest: protoManifestBlob, + Blobs: protoBlobs, Hash: commit.Hash().Hex(), Branch: branchName, Tags: tags, diff --git a/private/buf/bufsync/bufsynctest/bufsynctest.go b/private/buf/bufsync/bufsynctest/bufsynctest.go index 74825f6703..b9f5501c58 100644 --- a/private/buf/bufsync/bufsynctest/bufsynctest.go +++ b/private/buf/bufsync/bufsynctest/bufsynctest.go @@ -252,15 +252,15 @@ func doManualPushCommit( require.NoError(t, err) fileSet, err := bufcas.NewFileSetForBucket(context.Background(), builtModule.Bucket) require.NoError(t, err) - protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet) + protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet) require.NoError(t, err) handler.ManuallyPushModule( context.Background(), t, targetModuleIdentity, branch, - bufcasalpha.BlobToAlpha(protoManifestBlob), - bufcasalpha.BlobsToAlpha(protoBlobs), + protoManifestBlob, + protoBlobs, ) } func doManualPushRandomModule( @@ -278,15 +278,15 @@ func doManualPushRandomModule( require.NoError(t, err) fileSet, err := bufcas.NewFileSetForBucket(context.Background(), bucket) require.NoError(t, err) - protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet) + protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet) require.NoError(t, err) handler.ManuallyPushModule( context.Background(), t, targetModuleIdentity, branch, - bufcasalpha.BlobToAlpha(protoManifestBlob), - bufcasalpha.BlobsToAlpha(protoBlobs), + protoManifestBlob, + protoBlobs, ) } diff --git a/private/buf/bufsync/syncer_test.go b/private/buf/bufsync/syncer_test.go index 7d770d225d..f2a11bf14b 100644 --- a/private/buf/bufsync/syncer_test.go +++ b/private/buf/bufsync/syncer_test.go @@ -20,7 +20,6 @@ import ( "github.com/bufbuild/buf/private/buf/bufsync" "github.com/bufbuild/buf/private/buf/bufsync/bufsynctest" - "github.com/bufbuild/buf/private/bufpkg/bufcas" "github.com/bufbuild/buf/private/bufpkg/bufcas/bufcasalpha" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref" modulev1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/module/v1alpha1" @@ -286,7 +285,7 @@ func (c *testSyncHandler) ManuallyPushModule( manifest *modulev1alpha1.Blob, blobs []*modulev1alpha1.Blob, ) { - digest, err := bufcas.ProtoToDigest(bufcasalpha.AlphaToDigest(manifest.Digest)) + digest, err := bufcasalpha.AlphaToDigest(manifest.Digest) require.NoError(t, err) if branchName == "" { // release commit diff --git a/private/buf/cmd/buf/command/push/push.go b/private/buf/cmd/buf/command/push/push.go index 72973b1233..a49612afff 100644 --- a/private/buf/cmd/buf/command/push/push.go +++ b/private/buf/cmd/buf/command/push/push.go @@ -279,7 +279,7 @@ func push( if err != nil { return nil, err } - protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet) + protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet) if err != nil { return nil, err } @@ -293,8 +293,8 @@ func push( connect.NewRequest(®istryv1alpha1.PushManifestAndBlobsRequest{ Owner: moduleIdentity.Owner(), Repository: moduleIdentity.Repository(), - Manifest: bufcasalpha.BlobToAlpha(protoManifestBlob), - Blobs: bufcasalpha.BlobsToAlpha(protoBlobs), + Manifest: protoManifestBlob, + Blobs: protoBlobs, Tags: flags.Tags, DraftName: draftOrBranchName, }), diff --git a/private/buf/cmd/buf/command/push/push_test.go b/private/buf/cmd/buf/command/push/push_test.go index 314d003763..20360df567 100644 --- a/private/buf/cmd/buf/command/push/push_test.go +++ b/private/buf/cmd/buf/command/push/push_test.go @@ -30,7 +30,6 @@ import ( "sync" "testing" - storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1" "connectrpc.com/connect" "github.com/bufbuild/buf/private/buf/cmd/buf/internal/internaltesting" "github.com/bufbuild/buf/private/bufpkg/bufcas" @@ -139,11 +138,7 @@ func TestPushManifestIsSmallerBucket(t *testing.T) { request := mock.PushManifestRequest() require.NotNil(t, request) requestManifest := request.Manifest - manifest, err := bufcas.ProtoBlobToManifest( - bufcasalpha.AlphaToBlob( - requestManifest, - ), - ) + manifest, err := bufcasalpha.AlphaManifestBlobToManifest(requestManifest) require.NoError(t, err) assert.Nil(t, manifest.GetDigest("baz.file"), "baz.file should not be pushed") } @@ -161,17 +156,16 @@ func TestBucketBlobs(t *testing.T) { ctx := context.Background() fileSet, err := bufcas.NewFileSetForBucket(ctx, bucket) require.NoError(t, err) - _, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet) - require.NoError(t, err) - assert.Equal(t, 2, len(protoBlobs)) + blobs := fileSet.BlobSet().Blobs() + assert.Equal(t, 2, len(blobs)) digests := make(map[string]struct{}) - for _, protoBlob := range protoBlobs { + for _, blob := range blobs { assert.Equal( t, - storagev1beta1.Digest_TYPE_SHAKE256, - protoBlob.Digest.Type, + bufcas.DigestTypeShake256, + blob.Digest().Type(), ) - hexDigest := hex.EncodeToString(protoBlob.Digest.Value) + hexDigest := hex.EncodeToString(blob.Digest().Value()) assert.NotContains(t, digests, hexDigest, "duplicated blob") digests[hexDigest] = struct{}{} } diff --git a/private/bufpkg/bufapimodule/module_reader.go b/private/bufpkg/bufapimodule/module_reader.go index 7e798eade9..a42a1ed9ab 100644 --- a/private/bufpkg/bufapimodule/module_reader.go +++ b/private/bufpkg/bufapimodule/module_reader.go @@ -21,7 +21,6 @@ import ( "io/fs" "connectrpc.com/connect" - "github.com/bufbuild/buf/private/bufpkg/bufcas" "github.com/bufbuild/buf/private/bufpkg/bufcas/bufcasalpha" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref" @@ -72,10 +71,7 @@ func (m *moduleReader) GetModule(ctx context.Context, modulePin bufmoduleref.Mod if resp.Manifest == nil { return nil, errors.New("expected non-nil manifest") } - fileSet, err := bufcas.ProtoManifestBlobAndBlobsToFileSet( - bufcasalpha.AlphaToBlob(resp.Manifest), - bufcasalpha.AlphaToBlobs(resp.Blobs), - ) + fileSet, err := bufcasalpha.AlphaManifestBlobAndBlobsToFileSet(resp.Manifest, resp.Blobs) if err != nil { return nil, err } diff --git a/private/bufpkg/bufapimodule/module_reader_test.go b/private/bufpkg/bufapimodule/module_reader_test.go index 4c52f3540b..a659d78f05 100644 --- a/private/bufpkg/bufapimodule/module_reader_test.go +++ b/private/bufpkg/bufapimodule/module_reader_test.go @@ -172,17 +172,8 @@ func (fm filemap) apply(m *mockDownloadService) error { if err != nil { return err } - protoManifestBlob, err := bufcas.ManifestToProtoBlob(fileSet.Manifest()) - if err != nil { - return err - } - protoBlobs, err := bufcas.BlobSetToProtoBlobs(fileSet.BlobSet()) - if err != nil { - return err - } - m.manifestBlob = bufcasalpha.BlobToAlpha(protoManifestBlob) - m.blobs = bufcasalpha.BlobsToAlpha(protoBlobs) - return nil + m.manifestBlob, m.blobs, err = bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet) + return err } func withBlobsFromMap(files map[string][]byte) option { diff --git a/private/bufpkg/bufcas/bufcasalpha/bufcasalpha.go b/private/bufpkg/bufcas/bufcasalpha/bufcasalpha.go index 93c9b5f8c9..b20cdabfab 100644 --- a/private/bufpkg/bufcas/bufcasalpha/bufcasalpha.go +++ b/private/bufpkg/bufcas/bufcasalpha/bufcasalpha.go @@ -13,69 +13,55 @@ // limitations under the License. // Package bufcasalpha temporarily converts v1alpha1 API types to new API types. -// -// Minimal validation is done as this is assumed to be done by the bufcas package, -// and this allows us to have less error returning. package bufcasalpha import ( - storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1" + "bytes" + + "github.com/bufbuild/buf/private/bufpkg/bufcas" modulev1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/module/v1alpha1" ) var ( - digestTypeToAlpha = map[storagev1beta1.Digest_Type]modulev1alpha1.DigestType{ - storagev1beta1.Digest_TYPE_SHAKE256: modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256, + digestTypeToAlpha = map[bufcas.DigestType]modulev1alpha1.DigestType{ + bufcas.DigestTypeShake256: modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256, } - alphaToDigestType = map[modulev1alpha1.DigestType]storagev1beta1.Digest_Type{ - modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256: storagev1beta1.Digest_TYPE_SHAKE256, + alphaToDigestType = map[modulev1alpha1.DigestType]bufcas.DigestType{ + modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256: bufcas.DigestTypeShake256, } ) -func DigestToAlpha(digest *storagev1beta1.Digest) *modulev1alpha1.Digest { - if digest == nil { - return nil - } +func DigestToAlpha(digest bufcas.Digest) *modulev1alpha1.Digest { return &modulev1alpha1.Digest{ - DigestType: digestTypeToAlpha[digest.Type], - Digest: digest.Value, + DigestType: digestTypeToAlpha[digest.Type()], + Digest: digest.Value(), } } -func AlphaToDigest(alphaDigest *modulev1alpha1.Digest) *storagev1beta1.Digest { - if alphaDigest == nil { - return nil - } - return &storagev1beta1.Digest{ - Type: alphaToDigestType[alphaDigest.DigestType], - Value: alphaDigest.Digest, - } +func AlphaToDigest(alphaDigest *modulev1alpha1.Digest) (bufcas.Digest, error) { + return bufcas.NewDigest( + alphaDigest.GetDigest(), + bufcas.DigestWithDigestType(alphaToDigestType[alphaDigest.GetDigestType()]), + ) } -func BlobToAlpha(blob *storagev1beta1.Blob) *modulev1alpha1.Blob { - if blob == nil { - return nil - } +func BlobToAlpha(blob bufcas.Blob) *modulev1alpha1.Blob { return &modulev1alpha1.Blob{ - Digest: DigestToAlpha(blob.Digest), - Content: blob.Content, + Digest: DigestToAlpha(blob.Digest()), + Content: blob.Content(), } } -func AlphaToBlob(alphaBlob *modulev1alpha1.Blob) *storagev1beta1.Blob { - if alphaBlob == nil { - return nil - } - return &storagev1beta1.Blob{ - Digest: AlphaToDigest(alphaBlob.Digest), - Content: alphaBlob.Content, +func AlphaToBlob(alphaBlob *modulev1alpha1.Blob) (bufcas.Blob, error) { + digest, err := AlphaToDigest(alphaBlob.GetDigest()) + if err != nil { + return nil, err } + return bufcas.NewBlobForContent(bytes.NewReader(alphaBlob.GetContent()), bufcas.BlobWithKnownDigest(digest)) } -func BlobsToAlpha(blobs []*storagev1beta1.Blob) []*modulev1alpha1.Blob { - if blobs == nil { - return nil - } +func BlobSetToAlpha(blobSet bufcas.BlobSet) []*modulev1alpha1.Blob { + blobs := blobSet.Blobs() alphaBlobs := make([]*modulev1alpha1.Blob, len(blobs)) for i, blob := range blobs { alphaBlobs[i] = BlobToAlpha(blob) @@ -83,13 +69,57 @@ func BlobsToAlpha(blobs []*storagev1beta1.Blob) []*modulev1alpha1.Blob { return alphaBlobs } -func AlphaToBlobs(alphaBlobs []*modulev1alpha1.Blob) []*storagev1beta1.Blob { - if alphaBlobs == nil { - return nil - } - blobs := make([]*storagev1beta1.Blob, len(alphaBlobs)) +func AlphaToBlobSet(alphaBlobs []*modulev1alpha1.Blob) (bufcas.BlobSet, error) { + blobs := make([]bufcas.Blob, len(alphaBlobs)) + var err error for i, alphaBlob := range alphaBlobs { - blobs[i] = AlphaToBlob(alphaBlob) + blobs[i], err = AlphaToBlob(alphaBlob) + if err != nil { + return nil, err + } + } + return bufcas.NewBlobSet(blobs) +} + +func AlphaManifestBlobToManifest(manifestAlphaBlob *modulev1alpha1.Blob) (bufcas.Manifest, error) { + manifestBlob, err := AlphaToBlob(manifestAlphaBlob) + if err != nil { + return nil, err + } + return bufcas.BlobToManifest(manifestBlob) +} + +func ManifestToAlphaManifestBlob(manifest bufcas.Manifest) (*modulev1alpha1.Blob, error) { + manifestBlob, err := bufcas.ManifestToBlob(manifest) + if err != nil { + return nil, err + } + return BlobToAlpha(manifestBlob), nil +} + +func AlphaManifestBlobAndBlobsToFileSet( + manifestAlphaBlob *modulev1alpha1.Blob, + alphaBlobs []*modulev1alpha1.Blob, +) (bufcas.FileSet, error) { + manifestBlob, err := AlphaToBlob(manifestAlphaBlob) + if err != nil { + return nil, err + } + manifest, err := bufcas.BlobToManifest(manifestBlob) + if err != nil { + return nil, err + } + blobSet, err := AlphaToBlobSet(alphaBlobs) + if err != nil { + return nil, err + } + return bufcas.NewFileSet(manifest, blobSet) +} + +func FileSetToAlphaManifestBlobAndBlobs(fileset bufcas.FileSet) (*modulev1alpha1.Blob, []*modulev1alpha1.Blob, error) { + manifestBlob, err := bufcas.ManifestToBlob(fileset.Manifest()) + if err != nil { + return nil, nil, err } - return blobs + return BlobToAlpha(manifestBlob), BlobSetToAlpha(fileset.BlobSet()), nil } diff --git a/private/bufpkg/bufcas/digest.go b/private/bufpkg/bufcas/digest.go index 01b0724634..cb5eafef6e 100644 --- a/private/bufpkg/bufcas/digest.go +++ b/private/bufpkg/bufcas/digest.go @@ -20,11 +20,10 @@ import ( "errors" "fmt" "io" - "sort" "strconv" "strings" - storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1" + "github.com/bufbuild/buf/private/pkg/syserror" "golang.org/x/crypto/sha3" ) @@ -44,12 +43,6 @@ var ( stringToDigestType = map[string]DigestType{ "shake256": DigestTypeShake256, } - digestTypeToProto = map[DigestType]storagev1beta1.Digest_Type{ - DigestTypeShake256: storagev1beta1.Digest_TYPE_SHAKE256, - } - protoToDigestType = map[storagev1beta1.Digest_Type]DigestType{ - storagev1beta1.Digest_TYPE_SHAKE256: DigestTypeShake256, - } ) // DigestType is a type of digest. @@ -101,6 +94,21 @@ type Digest interface { isDigest() } +// NewDigest returns a new Digest for the value. +func NewDigest(value []byte, options ...DigestOption) (Digest, error) { + digestOptions := newDigestOptions() + for _, option := range options { + option(digestOptions) + } + if digestOptions.digestType == 0 { + digestOptions.digestType = DigestTypeShake256 + } + if err := validateDigestParameters(digestOptions.digestType, value); err != nil { + return nil, err + } + return newDigest(digestOptions.digestType, value), nil +} + // NewDigestForContent creates a new Digest based on the given content read from the Reader. // // A valid Digest is returned, even in the case of empty content. @@ -134,21 +142,8 @@ func NewDigestForContent(reader io.Reader, options ...DigestOption) (Digest, err return newDigest(DigestTypeShake256, value), nil default: // This is a system error. - return nil, fmt.Errorf("unknown DigestType: %v", digestOptions.digestType) - } -} - -// NewDigestForDigests returns a new Digest for the given Digests. -// -// Digests are sorted by string value, and then concatenated with newlines. The resulting -// content is then turned into a Digest. -func NewDigestForDigests(digests []Digest, options ...DigestOption) (Digest, error) { - digestStrings := make([]string, len(digests)) - for i, digest := range digests { - digestStrings[i] = digest.String() + return nil, syserror.Newf("unknown DigestType: %v", digestOptions.digestType) } - sort.Strings(digestStrings) - return NewDigestForContent(strings.NewReader(strings.Join(digestStrings, "\n")), options...) } // DigestOption is an option for a new Digest. @@ -281,7 +276,7 @@ func validateDigestParameters(digestType DigestType, value []byte) error { default: // This is really always a system error, but little harm in including it here, even // though it'll get converted into a ParseError in parse. - return fmt.Errorf(`unknown digest type: %q`, digestType.String()) + return syserror.Newf(`unknown digest type: %q`, digestType.String()) } return nil } diff --git a/private/bufpkg/bufcas/manifest.go b/private/bufpkg/bufcas/manifest.go index a8146e6296..80ca32224d 100644 --- a/private/bufpkg/bufcas/manifest.go +++ b/private/bufpkg/bufcas/manifest.go @@ -116,6 +116,13 @@ func ManifestToBlob(manifest Manifest) (Blob, error) { return NewBlobForContent(strings.NewReader(manifest.String())) } +// ManifestToDigest converts the string representation of the given Manifest into a Digest. +// +// The Manifest is assumed to be non-nil. +func ManifestToDigest(manifest Manifest) (Digest, error) { + return NewDigestForContent(strings.NewReader(manifest.String())) +} + // BlobToManifest converts the given Blob representing the string representation of a Manifest into a Manifest. // // # The Blob is assumed to be non-nil diff --git a/private/bufpkg/bufcas/parse_error.go b/private/bufpkg/bufcas/parse_error.go index e0b6a88232..a03edb5651 100644 --- a/private/bufpkg/bufcas/parse_error.go +++ b/private/bufpkg/bufcas/parse_error.go @@ -21,9 +21,6 @@ import ( // ParseError is an error that occurred during parsing. // // This is returned by all Parse.* functions in this package. -// -// This package should be the only package that creates ParseErrors. All fields are -// purposefully left private so that this is somewhat enforced. type ParseError struct { // typeString is the user-consumable string representing of the type that was attempted to be parsed. // diff --git a/private/bufpkg/bufcas/proto.go b/private/bufpkg/bufcas/proto.go deleted file mode 100644 index 9e64330062..0000000000 --- a/private/bufpkg/bufcas/proto.go +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufcas - -import ( - "bytes" - "fmt" - - storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1" - "github.com/bufbuild/buf/private/pkg/slicesext" - "github.com/bufbuild/protovalidate-go" - "google.golang.org/protobuf/proto" -) - -// This file contains public functions to convert to and from Protobuf types. -// -// TODO: One could argue we should not be validating input Protobuf messages, as these should -// be validated at a higher level. Output messages we likely want to validate, although we -// could define that these should be done elsewhere, but generally we like to validate at -// construction time. Consider removing at least input Protobuf message validation. - -var ( - // defaultValidate is the default Validator used in the absence of ProtoWithValidator - // being passed for a given Protobuf conversion function. - // - // Disabling validation by default for now by not instantiating this. - // TODO: discuss if we should do this by default, including on CLI side (probably not). - defaultValidator *protovalidate.Validator -) - -//func init() { -//var err error -//defaultValidator, err = protovalidate.New() -//if err != nil { -//panic(err.Error()) -//} -//} - -// BlobToProto converts the given Blob to a proto Blob. -func BlobToProto(blob Blob, options ...ProtoOption) (*storagev1beta1.Blob, error) { - protoDigest, err := DigestToProto(blob.Digest(), doNotProtoValidate(options)...) - if err != nil { - return nil, err - } - protoBlob := &storagev1beta1.Blob{ - Digest: protoDigest, - Content: blob.Content(), - } - if err := protoValidate(options, protoBlob); err != nil { - return nil, err - } - return protoBlob, nil -} - -// BlobsToProto converts the given Blobs to proto Blobs. -func BlobsToProto(blobs []Blob, options ...ProtoOption) ([]*storagev1beta1.Blob, error) { - return slicesext.MapError( - blobs, - func(blob Blob) (*storagev1beta1.Blob, error) { - return BlobToProto(blob, options...) - }, - ) -} - -// ProtoToBlob converts the given proto Blob to a Blob. -// -// Validation is performed to ensure that the Digest matches the computed Digest of the content. -func ProtoToBlob(protoBlob *storagev1beta1.Blob, options ...ProtoOption) (Blob, error) { - if err := protoValidate(options, protoBlob); err != nil { - return nil, err - } - digest, err := ProtoToDigest(protoBlob.Digest, doNotProtoValidate(options)...) - if err != nil { - return nil, err - } - return NewBlobForContent(bytes.NewReader(protoBlob.Content), BlobWithKnownDigest(digest)) -} - -// ProtoToBlobs converts the given proto Blobs to Blobs. -func ProtoToBlobs(protoBlobs []*storagev1beta1.Blob, options ...ProtoOption) ([]Blob, error) { - return slicesext.MapError( - protoBlobs, - func(protoBlob *storagev1beta1.Blob) (Blob, error) { - return ProtoToBlob(protoBlob, options...) - }, - ) -} - -// BlobSetToProtoBlobs converts the given BlobSet into proto Blobs. -func BlobSetToProtoBlobs(blobSet BlobSet, options ...ProtoOption) ([]*storagev1beta1.Blob, error) { - blobs := blobSet.Blobs() - protoBlobs := make([]*storagev1beta1.Blob, len(blobs)) - for i, blob := range blobs { - // Rely on validation in BlobToProto. - protoBlob, err := BlobToProto(blob, options...) - if err != nil { - return nil, err - } - protoBlobs[i] = protoBlob - } - return protoBlobs, nil -} - -// ProtoBlobsToBlobSet converts the given proto Blobs into a BlobSet. -func ProtoBlobsToBlobSet(protoBlobs []*storagev1beta1.Blob, options ...ProtoOption) (BlobSet, error) { - blobs := make([]Blob, len(protoBlobs)) - for i, protoBlob := range protoBlobs { - // Rely on validation in ProtoToBlob. - blob, err := ProtoToBlob(protoBlob, options...) - if err != nil { - return nil, err - } - blobs[i] = blob - } - return NewBlobSet(blobs) -} - -// DigestToProto converts the given Digest to a proto Digest. -func DigestToProto(digest Digest, options ...ProtoOption) (*storagev1beta1.Digest, error) { - protoDigestType, ok := digestTypeToProto[digest.Type()] - // Technically we have already done this validation but just to be safe. - if !ok { - return nil, fmt.Errorf("unknown DigestType: %v", digest.Type()) - } - protoDigest := &storagev1beta1.Digest{ - Type: protoDigestType, - Value: digest.Value(), - } - if err := protoValidate(options, protoDigest); err != nil { - return nil, err - } - return protoDigest, nil -} - -// DigestsToProto converts the given Digests to proto Digests. -func DigestsToProto(digests []Digest, options ...ProtoOption) ([]*storagev1beta1.Digest, error) { - return slicesext.MapError( - digests, - func(digest Digest) (*storagev1beta1.Digest, error) { - return DigestToProto(digest, options...) - }, - ) -} - -// ProtoToDigest converts the given proto Digest to a Digest. -// -// Validation is performed to ensure the DigestType is known, and the value -// is a valid digest value for the given DigestType. -func ProtoToDigest(protoDigest *storagev1beta1.Digest, options ...ProtoOption) (Digest, error) { - if err := protoValidate(options, protoDigest); err != nil { - return nil, err - } - digestType, ok := protoToDigestType[protoDigest.Type] - if !ok { - return nil, fmt.Errorf("unknown proto Digest.Type: %v", protoDigest.Type) - } - if err := validateDigestParameters(digestType, protoDigest.Value); err != nil { - return nil, err - } - return newDigest(digestType, protoDigest.Value), nil -} - -// ProtoToDigests converts the given proto Digests to Digests. -func ProtoToDigests(protoDigests []*storagev1beta1.Digest, options ...ProtoOption) ([]Digest, error) { - return slicesext.MapError( - protoDigests, - func(protoDigest *storagev1beta1.Digest) (Digest, error) { - return ProtoToDigest(protoDigest, options...) - }, - ) -} - -// FileNodeToProto converts the given FileNode to a proto FileNode. -func FileNodeToProto(fileNode FileNode, options ...ProtoOption) (*storagev1beta1.FileNode, error) { - protoDigest, err := DigestToProto(fileNode.Digest(), doNotProtoValidate(options)...) - if err != nil { - return nil, err - } - protoFileNode := &storagev1beta1.FileNode{ - Path: fileNode.Path(), - Digest: protoDigest, - } - if err := protoValidate(options, protoFileNode); err != nil { - return nil, err - } - return protoFileNode, nil -} - -// FileNodesToProto converts the given FileNodes to proto FileNodes. -func FileNodesToProto(fileNodes []FileNode, options ...ProtoOption) ([]*storagev1beta1.FileNode, error) { - return slicesext.MapError( - fileNodes, - func(fileNode FileNode) (*storagev1beta1.FileNode, error) { - return FileNodeToProto(fileNode, options...) - }, - ) -} - -// ProtoToFileNode converts the given proto FileNode to a FileNode. -// -// The path is validated to be normalized and non-empty. -func ProtoToFileNode(protoFileNode *storagev1beta1.FileNode, options ...ProtoOption) (FileNode, error) { - if err := protoValidate(options, protoFileNode); err != nil { - return nil, err - } - digest, err := ProtoToDigest(protoFileNode.Digest, doNotProtoValidate(options)...) - if err != nil { - return nil, err - } - return NewFileNode(protoFileNode.Path, digest) -} - -// ProtoToFileNodes converts the given proto FileNodes to FileNodes. -func ProtoToFileNodes(protoFileNodes []*storagev1beta1.FileNode, options ...ProtoOption) ([]FileNode, error) { - return slicesext.MapError( - protoFileNodes, - func(protoFileNode *storagev1beta1.FileNode) (FileNode, error) { - return ProtoToFileNode(protoFileNode, options...) - }, - ) -} - -// FileSetToProtoManifestBlobAndBlobs converts the given FileSet into a proto Blob representing the -// Manifest, and a set of Blobs representing the Files. -func FileSetToProtoManifestBlobAndBlobs( - fileSet FileSet, - options ...ProtoOption, -) (*storagev1beta1.Blob, []*storagev1beta1.Blob, error) { - // Rely on validation in ManifestToProtoBlob. - protoManifestBlob, err := ManifestToProtoBlob(fileSet.Manifest(), options...) - if err != nil { - return nil, nil, err - } - // Rely on validation in BlobSetToProtoBlobs. - protoBlobs, err := BlobSetToProtoBlobs(fileSet.BlobSet(), options...) - if err != nil { - return nil, nil, err - } - return protoManifestBlob, protoBlobs, nil -} - -// ProtoManifestBlobAndBlobsToFileSet converts the given manifest Blob and set of Blobs representing -// the Files into a FileSet. -// -// Validation is done to ensure the Manifest exactly matches the BlobSet. -func ProtoManifestBlobAndBlobsToFileSet( - protoManifestBlob *storagev1beta1.Blob, - protoBlobs []*storagev1beta1.Blob, - options ...ProtoOption, -) (FileSet, error) { - // Rely on validation in ProtoBlobToManifest. - manifest, err := ProtoBlobToManifest(protoManifestBlob, options...) - if err != nil { - return nil, err - } - // Rely on validation in ProtoBlobsToBlobSet. - blobSet, err := ProtoBlobsToBlobSet(protoBlobs, options...) - if err != nil { - return nil, err - } - return NewFileSet(manifest, blobSet) -} - -// ManifestToProtoBlob converts the string representation of the given Manifest into a proto Blob. -// -// The Manifest is assumed to be non-nil. -func ManifestToProtoBlob(manifest Manifest, options ...ProtoOption) (*storagev1beta1.Blob, error) { - blob, err := ManifestToBlob(manifest) - if err != nil { - return nil, err - } - // Rely on validation in BlobToProto. - return BlobToProto(blob, options...) -} - -// ProtoBlobToManifest converts the given proto Blob representing the string representation of a -// Manifest into a Manifest. -// -// The proto Blob is assumed to be non-nil. -// -// This function returns ParseErrors as it is effectively parsing the Manifest. -func ProtoBlobToManifest(protoBlob *storagev1beta1.Blob, options ...ProtoOption) (Manifest, error) { - // Rely on validation in ProtoToBlob. - blob, err := ProtoToBlob(protoBlob, options...) - if err != nil { - return nil, err - } - return BlobToManifest(blob) -} - -// ProtoOption is an option for a Protobuf conversion function. -type ProtoOption func(*protoOptions) - -// ProtoWithValidator says to use the given ProtoValidator. -// -// The default is to use a global instance of *protovalidator.Validator constructed for this package. -func ProtoWithValidator(validator ProtoValidator) ProtoOption { - return func(protoOptions *protoOptions) { - protoOptions.validator = validator - } -} - -// ProtoValidator is a validator for Protobuf messages. -type ProtoValidator interface { - Validate(proto.Message) error -} - -// *** PRIVATE *** - -type protoOptions struct { - validator ProtoValidator - doNotValidate bool -} - -func newProtoOptions() *protoOptions { - return &protoOptions{} -} - -// convenience function to do the validation we actually want based on the options values. -// -// If we have more ProtoOptions in the future, we can choose to instead pass *protoOptions -// instead of []ProtoOption, however the benefit will be minimal. -func protoValidate(options []ProtoOption, message proto.Message) error { - protoOptions := newProtoOptions() - for _, option := range options { - option(protoOptions) - } - if protoOptions.doNotValidate { - return nil - } - if protoOptions.validator != nil { - return protoOptions.validator.Validate(message) - } - if defaultValidator != nil { - // Need to do this as opposed to setting protoOptions.validator and - // then checking if protoOptions.Validator != nil because of Golang - // interface nil weirdness. - return defaultValidator.Validate(message) - } - return nil -} - -// convenience function to not do validation when we call Protobuf conversion functions within -// other Protobuf conversion functions. -// -// If we were to just blanket pass the options recursively, we would get double validation, as -// validation itself is recursive. This avoids that for calls within this package. -// -// Having this option allows us to add future ProtoOptions with less chance of introducing a bug, -// ie the call path is the same. -func doNotProtoValidate(options []ProtoOption) []ProtoOption { - return append(options, doNotProtoValidateOption) -} - -func doNotProtoValidateOption(protoOptions *protoOptions) { - // We cannot simply set validator to nil, as options can be passed in any order. - // Technically we control the order within this package but this deals with that. - protoOptions.doNotValidate = true -} diff --git a/private/pkg/syserror/syserror.go b/private/pkg/syserror/syserror.go new file mode 100644 index 0000000000..d16b25755d --- /dev/null +++ b/private/pkg/syserror/syserror.go @@ -0,0 +1,92 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package syserror handles "system errors". +// +// A system error is an error that should never actually happen, and should be +// propogated to the user as a bug in the codebase. System errors are generally +// defensive-programming assertions that we want to check for in our codebase. +// +// If a system error occurs, you may want to send a specialized help that says +// how to file a bug report. +package syserror + +import ( + "errors" + "fmt" +) + +// Error is a system error. +type Error struct { + Underlying error +} + +// Error implements error. +func (e *Error) Error() string { + if e == nil { + return "" + } + if e.Underlying == nil { + return "" + } + return "system error: " + e.Underlying.Error() +} + +// Unwrap implements errors.Unwrap for Error. +func (e *Error) Unwrap() error { + if e == nil { + return nil + } + return e.Underlying +} + +// New is a convenience function that returns a new system error by calling errors.new. +func New(text string) *Error { + return &Error{ + Underlying: errors.New(text), + } +} + +// Newf is a convenience function that returns a new system error by calling fmt.Errorf. +func Newf(format string, args ...any) *Error { + return &Error{ + Underlying: fmt.Errorf(format, args...), + } +} + +// Wrap returns a new system error for err. +// +// If err is already a system error, this returns err. +func Wrap(err error) error { + if Is(err) { + return err + } + return &Error{ + Underlying: err, + } +} + +// Is is a convenience function that returns true if err is a system error. +func Is(err error) bool { + _, ok := As(err) + return ok +} + +// As is a convenience function that returns err as an Error and true if err is an Error, +// and err and false otherwise. +func As(err error) (*Error, bool) { + target := &Error{} + ok := errors.As(err, &target) + return target, ok +} diff --git a/private/pkg/syserror/usage.gen.go b/private/pkg/syserror/usage.gen.go new file mode 100644 index 0000000000..85fb1993ba --- /dev/null +++ b/private/pkg/syserror/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package syserror + +import _ "github.com/bufbuild/buf/private/usage"