Update to latest bufbuild/registry protos (#2697)

Update to support the latest changes to bufbuild/registry. Migrate the
bufcasalpha package to use the bufcas interface types directly (since
the storagev1beta1 types are gone).

go.mod

@@ -4,7 +4,6 @@ go 1.19
 
 require (
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1
-	buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1
 	connectrpc.com/connect v1.14.0
 	connectrpc.com/otelconnect v0.6.0
 	github.com/bufbuild/protocompile v0.7.1

go.sum

@@ -1,7 +1,5 @@
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1 h1:u0olL4yf2p7Tl5jfsAK5keaFi+JFJuv1CDHrbiXkxkk=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20231115204500-e097f827e652.1/go.mod h1:tiTMKD8j6Pd/D2WzREoweufjzaJKHZg35f/VGcZ2v3I=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1 h1:IOqyoSoI4xrCGUc1PBdr7rbDUMEYX7fondG+fdDDvMo=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.32.0-20231205222057-ac336d436f46.1/go.mod h1:L+pboYag9Pd0vt+ErAHa7QSdqP0Dzd7S+4OLKlrXNXQ=
 connectrpc.com/connect v1.14.0 h1:PDS+J7uoz5Oui2VEOMcfz6Qft7opQM9hPiKvtGC01pA=
 connectrpc.com/connect v1.14.0/go.mod h1:uoAq5bmhhn43TwhaKdGKN/bZcGtzPW1v+ngDTn5u+8s=
 connectrpc.com/otelconnect v0.6.0 h1:VJAdQL9+sgdUw9+7+J+jq8pQo/h1S7tSFv2+vDcR7bU=

make/buf/all.mk

@@ -1,9 +1,5 @@
 GO_ALL_REPO_PKGS := ./cmd/... ./private/...
-# Keep bufbuild/registry pinned at version prior to landing the refactor branch.
-# This allows main to continue to build until it is ready to incorporate changes from the refactor branch.
-BUFBUILD_REGISTRY_VERSION := 20231205222057-ac336d436f46
-GO_GET_PKGS := $(GO_GET_PKGS) \
-	buf.build/gen/go/bufbuild/registry/protocolbuffers/go@v1.32.0-$(BUFBUILD_REGISTRY_VERSION).1
+#GO_GET_PKGS := $(GO_GET_PKGS)
 GO_BINS := $(GO_BINS) \
 	cmd/buf \
 	cmd/protoc-gen-buf-breaking \

private/buf/bufsync/bufsyncapi/sync_handler.go

@@ -447,15 +447,15 @@ func (h *syncHandler) syncCommitModule(
 	if err != nil {
 		return nil, err
 	}
-	protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet)
+	protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet)
 	if err != nil {
 		return nil, err
 	}
 	resp, err := service.SyncGitCommit(ctx, connect.NewRequest(&registryv1alpha1.SyncGitCommitRequest{
 		Owner:      moduleIdentity.Owner(),
 		Repository: moduleIdentity.Repository(),
-		Manifest:   bufcasalpha.BlobToAlpha(protoManifestBlob),
-		Blobs:      bufcasalpha.BlobsToAlpha(protoBlobs),
+		Manifest:   protoManifestBlob,
+		Blobs:      protoBlobs,
 		Hash:       commit.Hash().Hex(),
 		Branch:     branchName,
 		Tags:       tags,

private/buf/bufsync/bufsynctest/bufsynctest.go

@@ -252,15 +252,15 @@ func doManualPushCommit(
 	require.NoError(t, err)
 	fileSet, err := bufcas.NewFileSetForBucket(context.Background(), builtModule.Bucket)
 	require.NoError(t, err)
-	protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet)
+	protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet)
 	require.NoError(t, err)
 	handler.ManuallyPushModule(
 		context.Background(),
 		t,
 		targetModuleIdentity,
 		branch,
-		bufcasalpha.BlobToAlpha(protoManifestBlob),
-		bufcasalpha.BlobsToAlpha(protoBlobs),
+		protoManifestBlob,
+		protoBlobs,
 	)
 }
 func doManualPushRandomModule(
@@ -278,15 +278,15 @@ func doManualPushRandomModule(
 	require.NoError(t, err)
 	fileSet, err := bufcas.NewFileSetForBucket(context.Background(), bucket)
 	require.NoError(t, err)
-	protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet)
+	protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet)
 	require.NoError(t, err)
 	handler.ManuallyPushModule(
 		context.Background(),
 		t,
 		targetModuleIdentity,
 		branch,
-		bufcasalpha.BlobToAlpha(protoManifestBlob),
-		bufcasalpha.BlobsToAlpha(protoBlobs),
+		protoManifestBlob,
+		protoBlobs,
 	)
 }
 

private/buf/bufsync/syncer_test.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/bufbuild/buf/private/buf/bufsync"
 	"github.com/bufbuild/buf/private/buf/bufsync/bufsynctest"
-	"github.com/bufbuild/buf/private/bufpkg/bufcas"
 	"github.com/bufbuild/buf/private/bufpkg/bufcas/bufcasalpha"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref"
 	modulev1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/module/v1alpha1"
@@ -286,7 +285,7 @@ func (c *testSyncHandler) ManuallyPushModule(
 	manifest *modulev1alpha1.Blob,
 	blobs []*modulev1alpha1.Blob,
 ) {
-	digest, err := bufcas.ProtoToDigest(bufcasalpha.AlphaToDigest(manifest.Digest))
+	digest, err := bufcasalpha.AlphaToDigest(manifest.Digest)
 	require.NoError(t, err)
 	if branchName == "" {
 		// release commit

private/buf/cmd/buf/command/push/push.go

@@ -279,7 +279,7 @@ func push(
 	if err != nil {
 		return nil, err
 	}
-	protoManifestBlob, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet)
+	protoManifestBlob, protoBlobs, err := bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet)
 	if err != nil {
 		return nil, err
 	}
@@ -293,8 +293,8 @@ func push(
 		connect.NewRequest(&registryv1alpha1.PushManifestAndBlobsRequest{
 			Owner:      moduleIdentity.Owner(),
 			Repository: moduleIdentity.Repository(),
-			Manifest:   bufcasalpha.BlobToAlpha(protoManifestBlob),
-			Blobs:      bufcasalpha.BlobsToAlpha(protoBlobs),
+			Manifest:   protoManifestBlob,
+			Blobs:      protoBlobs,
 			Tags:       flags.Tags,
 			DraftName:  draftOrBranchName,
 		}),

private/buf/cmd/buf/command/push/push_test.go

@@ -30,7 +30,6 @@ import (
 	"sync"
 	"testing"
 
-	storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1"
 	"connectrpc.com/connect"
 	"github.com/bufbuild/buf/private/buf/cmd/buf/internal/internaltesting"
 	"github.com/bufbuild/buf/private/bufpkg/bufcas"
@@ -139,11 +138,7 @@ func TestPushManifestIsSmallerBucket(t *testing.T) {
 	request := mock.PushManifestRequest()
 	require.NotNil(t, request)
 	requestManifest := request.Manifest
-	manifest, err := bufcas.ProtoBlobToManifest(
-		bufcasalpha.AlphaToBlob(
-			requestManifest,
-		),
-	)
+	manifest, err := bufcasalpha.AlphaManifestBlobToManifest(requestManifest)
 	require.NoError(t, err)
 	assert.Nil(t, manifest.GetDigest("baz.file"), "baz.file should not be pushed")
 }
@@ -161,17 +156,16 @@ func TestBucketBlobs(t *testing.T) {
 	ctx := context.Background()
 	fileSet, err := bufcas.NewFileSetForBucket(ctx, bucket)
 	require.NoError(t, err)
-	_, protoBlobs, err := bufcas.FileSetToProtoManifestBlobAndBlobs(fileSet)
-	require.NoError(t, err)
-	assert.Equal(t, 2, len(protoBlobs))
+	blobs := fileSet.BlobSet().Blobs()
+	assert.Equal(t, 2, len(blobs))
 	digests := make(map[string]struct{})
-	for _, protoBlob := range protoBlobs {
+	for _, blob := range blobs {
 		assert.Equal(
 			t,
-			storagev1beta1.Digest_TYPE_SHAKE256,
-			protoBlob.Digest.Type,
+			bufcas.DigestTypeShake256,
+			blob.Digest().Type(),
 		)
-		hexDigest := hex.EncodeToString(protoBlob.Digest.Value)
+		hexDigest := hex.EncodeToString(blob.Digest().Value())
 		assert.NotContains(t, digests, hexDigest, "duplicated blob")
 		digests[hexDigest] = struct{}{}
 	}

private/bufpkg/bufapimodule/module_reader.go

@@ -21,7 +21,6 @@ import (
 	"io/fs"
 
 	"connectrpc.com/connect"
-	"github.com/bufbuild/buf/private/bufpkg/bufcas"
 	"github.com/bufbuild/buf/private/bufpkg/bufcas/bufcasalpha"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref"
@@ -72,10 +71,7 @@ func (m *moduleReader) GetModule(ctx context.Context, modulePin bufmoduleref.Mod
 	if resp.Manifest == nil {
 		return nil, errors.New("expected non-nil manifest")
 	}
-	fileSet, err := bufcas.ProtoManifestBlobAndBlobsToFileSet(
-		bufcasalpha.AlphaToBlob(resp.Manifest),
-		bufcasalpha.AlphaToBlobs(resp.Blobs),
-	)
+	fileSet, err := bufcasalpha.AlphaManifestBlobAndBlobsToFileSet(resp.Manifest, resp.Blobs)
 	if err != nil {
 		return nil, err
 	}

private/bufpkg/bufapimodule/module_reader_test.go

@@ -172,17 +172,8 @@ func (fm filemap) apply(m *mockDownloadService) error {
 	if err != nil {
 		return err
 	}
-	protoManifestBlob, err := bufcas.ManifestToProtoBlob(fileSet.Manifest())
-	if err != nil {
-		return err
-	}
-	protoBlobs, err := bufcas.BlobSetToProtoBlobs(fileSet.BlobSet())
-	if err != nil {
-		return err
-	}
-	m.manifestBlob = bufcasalpha.BlobToAlpha(protoManifestBlob)
-	m.blobs = bufcasalpha.BlobsToAlpha(protoBlobs)
-	return nil
+	m.manifestBlob, m.blobs, err = bufcasalpha.FileSetToAlphaManifestBlobAndBlobs(fileSet)
+	return err
 }
 
 func withBlobsFromMap(files map[string][]byte) option {

private/bufpkg/bufcas/bufcasalpha/bufcasalpha.go

@@ -13,83 +13,113 @@
 // limitations under the License.
 
 // Package bufcasalpha temporarily converts v1alpha1 API types to new API types.
-//
-// Minimal validation is done as this is assumed to be done by the bufcas package,
-// and this allows us to have less error returning.
 package bufcasalpha
 
 import (
-	storagev1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/storage/v1beta1"
+	"bytes"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufcas"
 	modulev1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/module/v1alpha1"
 )
 
 var (
-	digestTypeToAlpha = map[storagev1beta1.Digest_Type]modulev1alpha1.DigestType{
-		storagev1beta1.Digest_TYPE_SHAKE256: modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256,
+	digestTypeToAlpha = map[bufcas.DigestType]modulev1alpha1.DigestType{
+		bufcas.DigestTypeShake256: modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256,
 	}
-	alphaToDigestType = map[modulev1alpha1.DigestType]storagev1beta1.Digest_Type{
-		modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256: storagev1beta1.Digest_TYPE_SHAKE256,
+	alphaToDigestType = map[modulev1alpha1.DigestType]bufcas.DigestType{
+		modulev1alpha1.DigestType_DIGEST_TYPE_SHAKE256: bufcas.DigestTypeShake256,
 	}
 )
 
-func DigestToAlpha(digest *storagev1beta1.Digest) *modulev1alpha1.Digest {
-	if digest == nil {
-		return nil
-	}
+func DigestToAlpha(digest bufcas.Digest) *modulev1alpha1.Digest {
 	return &modulev1alpha1.Digest{
-		DigestType: digestTypeToAlpha[digest.Type],
-		Digest:     digest.Value,
+		DigestType: digestTypeToAlpha[digest.Type()],
+		Digest:     digest.Value(),
 	}
 }
 
-func AlphaToDigest(alphaDigest *modulev1alpha1.Digest) *storagev1beta1.Digest {
-	if alphaDigest == nil {
-		return nil
-	}
-	return &storagev1beta1.Digest{
-		Type:  alphaToDigestType[alphaDigest.DigestType],
-		Value: alphaDigest.Digest,
-	}
+func AlphaToDigest(alphaDigest *modulev1alpha1.Digest) (bufcas.Digest, error) {
+	return bufcas.NewDigest(
+		alphaDigest.GetDigest(),
+		bufcas.DigestWithDigestType(alphaToDigestType[alphaDigest.GetDigestType()]),
+	)
 }
 
-func BlobToAlpha(blob *storagev1beta1.Blob) *modulev1alpha1.Blob {
-	if blob == nil {
-		return nil
-	}
+func BlobToAlpha(blob bufcas.Blob) *modulev1alpha1.Blob {
 	return &modulev1alpha1.Blob{
-		Digest:  DigestToAlpha(blob.Digest),
-		Content: blob.Content,
+		Digest:  DigestToAlpha(blob.Digest()),
+		Content: blob.Content(),
 	}
 }
 
-func AlphaToBlob(alphaBlob *modulev1alpha1.Blob) *storagev1beta1.Blob {
-	if alphaBlob == nil {
-		return nil
-	}
-	return &storagev1beta1.Blob{
-		Digest:  AlphaToDigest(alphaBlob.Digest),
-		Content: alphaBlob.Content,
+func AlphaToBlob(alphaBlob *modulev1alpha1.Blob) (bufcas.Blob, error) {
+	digest, err := AlphaToDigest(alphaBlob.GetDigest())
+	if err != nil {
+		return nil, err
 	}
+	return bufcas.NewBlobForContent(bytes.NewReader(alphaBlob.GetContent()), bufcas.BlobWithKnownDigest(digest))
 }
 
-func BlobsToAlpha(blobs []*storagev1beta1.Blob) []*modulev1alpha1.Blob {
-	if blobs == nil {
-		return nil
-	}
+func BlobSetToAlpha(blobSet bufcas.BlobSet) []*modulev1alpha1.Blob {
+	blobs := blobSet.Blobs()
 	alphaBlobs := make([]*modulev1alpha1.Blob, len(blobs))
 	for i, blob := range blobs {
 		alphaBlobs[i] = BlobToAlpha(blob)
 	}
 	return alphaBlobs
 }
 
-func AlphaToBlobs(alphaBlobs []*modulev1alpha1.Blob) []*storagev1beta1.Blob {
-	if alphaBlobs == nil {
-		return nil
-	}
-	blobs := make([]*storagev1beta1.Blob, len(alphaBlobs))
+func AlphaToBlobSet(alphaBlobs []*modulev1alpha1.Blob) (bufcas.BlobSet, error) {
+	blobs := make([]bufcas.Blob, len(alphaBlobs))
+	var err error
 	for i, alphaBlob := range alphaBlobs {
-		blobs[i] = AlphaToBlob(alphaBlob)
+		blobs[i], err = AlphaToBlob(alphaBlob)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return bufcas.NewBlobSet(blobs)
+}
+
+func AlphaManifestBlobToManifest(manifestAlphaBlob *modulev1alpha1.Blob) (bufcas.Manifest, error) {
+	manifestBlob, err := AlphaToBlob(manifestAlphaBlob)
+	if err != nil {
+		return nil, err
+	}
+	return bufcas.BlobToManifest(manifestBlob)
+}
+
+func ManifestToAlphaManifestBlob(manifest bufcas.Manifest) (*modulev1alpha1.Blob, error) {
+	manifestBlob, err := bufcas.ManifestToBlob(manifest)
+	if err != nil {
+		return nil, err
+	}
+	return BlobToAlpha(manifestBlob), nil
+}
+
+func AlphaManifestBlobAndBlobsToFileSet(
+	manifestAlphaBlob *modulev1alpha1.Blob,
+	alphaBlobs []*modulev1alpha1.Blob,
+) (bufcas.FileSet, error) {
+	manifestBlob, err := AlphaToBlob(manifestAlphaBlob)
+	if err != nil {
+		return nil, err
+	}
+	manifest, err := bufcas.BlobToManifest(manifestBlob)
+	if err != nil {
+		return nil, err
+	}
+	blobSet, err := AlphaToBlobSet(alphaBlobs)
+	if err != nil {
+		return nil, err
+	}
+	return bufcas.NewFileSet(manifest, blobSet)
+}
+
+func FileSetToAlphaManifestBlobAndBlobs(fileset bufcas.FileSet) (*modulev1alpha1.Blob, []*modulev1alpha1.Blob, error) {
+	manifestBlob, err := bufcas.ManifestToBlob(fileset.Manifest())
+	if err != nil {
+		return nil, nil, err
 	}
-	return blobs
+	return BlobToAlpha(manifestBlob), BlobSetToAlpha(fileset.BlobSet()), nil
 }