Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix merge main -> bufmod #2628

Merged
merged 5 commits into from
Nov 29, 2023
Merged

Fix merge main -> bufmod #2628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7f29771
Split create repo and sync module into different steps (#2624)
unmultimedio Nov 27, 2023
4eab82b
Bump buf.build/gen/go/bufbuild/registry/protocolbuffers/go from 1.31.…
dependabot[bot] Nov 27, 2023
57a9b07
Add support for C language plugins (#2626)
pkwarren Nov 28, 2023
f1acbea
Merge remote-tracking branch 'origin/main' into bufmod
unmultimedio Nov 28, 2023
9e97711
merge conflicts
unmultimedio Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 37 additions & 65 deletions private/buf/bufsync/bufsyncapi/sync_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"github.com/bufbuild/buf/private/bufpkg/bufcas"
"github.com/bufbuild/buf/private/bufpkg/bufcas/bufcasalpha"
"github.com/bufbuild/buf/private/bufpkg/bufmodule"
"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref"

Check failure on line 26 in private/buf/bufsync/bufsyncapi/sync_handler.go

View workflow job for this annotation

GitHub Actions / test 

no required module provides package github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref; to add it:

Check failure on line 26 in private/buf/bufsync/bufsyncapi/sync_handler.go

View workflow job for this annotation

GitHub Actions / test-previous 

no required module provides package github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref; to add it:

Check failure on line 26 in private/buf/bufsync/bufsyncapi/sync_handler.go

View workflow job for this annotation

GitHub Actions / test 

no required module provides package github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref; to add it:
"github.com/bufbuild/buf/private/gen/proto/connect/buf/alpha/registry/v1alpha1/registryv1alpha1connect"
registryv1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/registry/v1alpha1"
"github.com/bufbuild/buf/private/pkg/app/appflag"
Expand Down Expand Up @@ -54,6 +55,7 @@

moduleFullNameToRepositoryIDCache map[string]string
moduleFullNameToDefaultBranchCache map[string]string
existingModuleFullNameCache map[string]struct{}
}

func newSyncHandler(
Expand All @@ -75,6 +77,7 @@
createWithVisibility: createWithVisibility,
moduleFullNameToRepositoryIDCache: make(map[string]string),
moduleFullNameToDefaultBranchCache: make(map[string]string),
existingModuleFullNameCache: make(map[string]struct{}),
syncServiceClientFactory: syncServiceClientFactory,
referenceServiceClientFactory: referenceServiceClientFactory,
repositoryServiceClientFactory: repositoryServiceClientFactory,
Expand Down Expand Up @@ -184,19 +187,19 @@
if err != nil {
return err
}
referenceService := h.referenceServiceClientFactory(moduleTags.TargetModuleFullName().Remote())
repositoryTagService := h.repositoryTagServiceClientFactory(moduleTags.TargetModuleFullName().Remote())
referenceService := h.referenceServiceClientFactory(moduleTags.TargetModuleFullName().Registry())
repositoryTagService := h.repositoryTagServiceClientFactory(moduleTags.TargetModuleFullName().Registry())
commitRes, err := referenceService.GetReferenceByName(ctx, connect.NewRequest(&registryv1alpha1.GetReferenceByNameRequest{
Owner: moduleTags.TargetModuleFullName().Owner(),
RepositoryName: moduleTags.TargetModuleFullName().Repository(),
RepositoryName: moduleTags.TargetModuleFullName().Name(),
Name: commit.Commit().Hash().Hex(),
}))
if err != nil {
if connect.CodeOf(err) == connect.CodeNotFound {
return fmt.Errorf(
"git commit %q is not known to module %q",
commit.Commit().Hash(),
moduleTags.TargetModuleFullName().IdentityString(),
moduleTags.TargetModuleFullName().String(),
)
}
return fmt.Errorf("get reference by name %q: %w", commit.Commit().Hash(), err)
Expand All @@ -205,7 +208,7 @@
return fmt.Errorf(
"git commit %q is not synced to module %q",
commit.Commit().Hash(),
moduleTags.TargetModuleFullName().IdentityString(),
moduleTags.TargetModuleFullName().String(),
)
}
for _, tag := range commit.Tags() {
Expand All @@ -220,7 +223,7 @@
CommitName: commitRes.Msg.Reference.GetVcsCommit().CommitName,
}))
if err != nil {
return fmt.Errorf("create new tag %q on module %q: %w", tag, moduleTags.TargetModuleFullName().IdentityString(), err)
return fmt.Errorf("create new tag %q on module %q: %w", tag, moduleTags.TargetModuleFullName().String(), err)
}
} else {
// TODO: don't do this unless we need to
Expand All @@ -230,7 +233,7 @@
CommitName: &commitRes.Msg.Reference.GetVcsCommit().CommitName,
}))
if err != nil {
return fmt.Errorf("update existing tag %q on module %q: %w", tag, moduleTags.TargetModuleFullName().IdentityString(), err)
return fmt.Errorf("update existing tag %q on module %q: %w", tag, moduleTags.TargetModuleFullName().String(), err)
}
}
}
Expand All @@ -244,7 +247,12 @@
if err != nil {
return fmt.Errorf("read bucket for commit %q: %w", moduleCommit.Commit().Hash(), err)
}
syncPoint, err := h.pushOrCreate(
if h.createWithVisibility != nil {
if err := h.createRepository(ctx, moduleBranch.TargetModuleFullName()); err != nil {
return fmt.Errorf("create repo %s: %w", moduleBranch.TargetModuleFullName().String(), err)
}
}
syncPoint, err := h.syncCommitModule(
ctx,
moduleCommit.Commit(),
moduleBranch.BranchName(),
Expand All @@ -253,26 +261,26 @@
bucket,
)
if err != nil {
// We failed to push. We fail hard on this because the error may be recoverable
// We failed to sync. We fail hard on this because the error may be recoverable
// (i.e., the BSR may be down) and we should re-attempt this commit.
return fmt.Errorf(
"failed to push or create %s at %s: %w",
moduleBranch.TargetModuleFullName().IdentityString(),
"sync module %s at branch %s commit %s directory %s: %w",
moduleBranch.TargetModuleFullName().String(),
moduleBranch.BranchName(),
moduleCommit.Commit().Hash(),
moduleBranch.Directory(),
err,
)
}
_, err = h.container.Stderr().Write([]byte(
syncMsg := fmt.Sprintf(
// from local -> to remote
// <module-directory>:<git-branch>:<git-commit-hash> -> <module-identity>:<bsr-commit-name>
fmt.Sprintf(
"%s:%s:%s -> %s:%s\n",
moduleBranch.Directory(), moduleBranch.BranchName(), moduleCommit.Commit().Hash().Hex(),
moduleBranch.TargetModuleFullName().IdentityString(), syncPoint.BsrCommitName,
)),
"%s:%s:%s -> %s:%s\n",
moduleBranch.Directory(), moduleBranch.BranchName(), moduleCommit.Commit().Hash().Hex(),
moduleBranch.TargetModuleFullName().String(), syncPoint.BsrCommitName,
)
if err != nil {
return err
if _, err := h.container.Stderr().Write([]byte(syncMsg)); err != nil {
return fmt.Errorf("write %q to stderr: %w", syncMsg, err)
}
}
return nil
Expand Down Expand Up @@ -427,48 +435,7 @@
return true, nil
}

func (h *syncHandler) pushOrCreate(
ctx context.Context,
commit git.Commit,
branchName string,
tags []string,
moduleFullName bufmodule.ModuleFullName,
moduleBucket storage.ReadBucket,
) (*registryv1alpha1.GitSyncPoint, error) {
modulePin, err := h.push(
ctx,
commit,
branchName,
tags,
moduleFullName,
moduleBucket,
)
if err != nil {
// We rely on Push* returning a NotFound error to denote the repository is not created.
// This technically could be a NotFound error for some other entity than the repository
// in question, however if it is, then this Create call will just fail as the repository
// is already created, and there is no side effect. The 99% case is that a NotFound
// error is because the repository does not exist, and we want to avoid having to do
// a GetRepository RPC call for every call to push --create.
if h.createWithVisibility != nil && connect.CodeOf(err) == connect.CodeNotFound {
if err := h.create(ctx, moduleFullName); err != nil {
return nil, fmt.Errorf("create repo: %w", err)
}
return h.push(
ctx,
commit,
branchName,
tags,
moduleFullName,
moduleBucket,
)
}
return nil, fmt.Errorf("push: %w", err)
}
return modulePin, nil
}

func (h *syncHandler) push(
func (h *syncHandler) syncCommitModule(
ctx context.Context,
commit git.Commit,
branchName string,
Expand Down Expand Up @@ -510,10 +477,13 @@
return resp.Msg.SyncPoint, nil
}

func (h *syncHandler) create(
func (h *syncHandler) createRepository(
ctx context.Context,
moduleFullName bufmodule.ModuleFullName,
) error {
if _, alreadyExists := h.existingModuleFullNameCache[moduleFullName.String()]; alreadyExists {
return nil
}
service := h.repositoryServiceClientFactory(moduleFullName.Registry())
fullName := moduleFullName.Owner() + "/" + moduleFullName.Name()
_, err := service.CreateRepositoryByFullName(
Expand All @@ -523,8 +493,10 @@
Visibility: *h.createWithVisibility,
}),
)
if err != nil && connect.CodeOf(err) == connect.CodeAlreadyExists {
return connect.NewError(connect.CodeInternal, fmt.Errorf("expected repository %s to be missing but found the repository to already exist", fullName))
if err != nil && connect.CodeOf(err) != connect.CodeAlreadyExists {
return err
}
return err
// if created successfully or if it already existed, cache it
h.existingModuleFullNameCache[moduleFullName.String()] = struct{}{}
return nil
}
3 changes: 2 additions & 1 deletion private/bufpkg/bufplugin/bufplugin_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,13 @@ func TestLanguagesToProtoLanguages(t *testing.T) {
},
protoLanguages,
)
protoLanguages, err = OutputLanguagesToProtoLanguages([]string{"java", "kotlin"})
protoLanguages, err = OutputLanguagesToProtoLanguages([]string{"java", "kotlin", "c"})
require.NoError(t, err)
assert.Equal(t,
[]registryv1alpha1.PluginLanguage{
registryv1alpha1.PluginLanguage_PLUGIN_LANGUAGE_JAVA,
registryv1alpha1.PluginLanguage_PLUGIN_LANGUAGE_KOTLIN,
registryv1alpha1.PluginLanguage_PLUGIN_LANGUAGE_C,
},
protoLanguages,
)
Expand Down
Loading
Loading