Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add buf migrate command #2669

Merged
merged 41 commits into from
Dec 27, 2023
Merged
Show file tree
Hide file tree
Changes from 39 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
3980daf
file name cleanup
oliversun9 Dec 11, 2023
736f424
resolve, probably dosen't work because it hasn't been tried with the …
oliversun9 Dec 12, 2023
eb62f9a
doesn't compile
oliversun9 Dec 12, 2023
c176f36
add FileName to buf lock, buf yaml and buf work yaml
oliversun9 Dec 12, 2023
2f2836f
file name is empty unless caller specifies non-empty file name
oliversun9 Dec 12, 2023
3f7d3c9
update comments
oliversun9 Dec 12, 2023
24634df
comment
oliversun9 Dec 12, 2023
2c9e67e
Merge branch 'osun/file-name' into osun/migrate
oliversun9 Dec 12, 2023
23ed5a5
commit
oliversun9 Dec 12, 2023
0305eaf
fix
oliversun9 Dec 12, 2023
1edab45
Merge branch 'osun/file-name' into osun/migrate
oliversun9 Dec 12, 2023
2a6f3cf
migrate generation
oliversun9 Dec 12, 2023
66183e7
update resolve
oliversun9 Dec 13, 2023
95e6fce
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 13, 2023
ab1956f
update flag names
oliversun9 Dec 13, 2023
0d4d31e
dont create buf.lock if we haven't seen one
oliversun9 Dec 13, 2023
72a63d2
field name
oliversun9 Dec 13, 2023
bc6eb65
handle module without a config correctly
oliversun9 Dec 13, 2023
2c841f7
only remove workspace modules from the new buf.lock
oliversun9 Dec 13, 2023
51c3517
move migrate out of beta
oliversun9 Dec 13, 2023
dfcb3f4
update command description
oliversun9 Dec 14, 2023
a3616d5
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 14, 2023
e81555b
if workspace dir is the only one specified, destination dir is worksp…
oliversun9 Dec 14, 2023
7d7101d
remove some TODOs
oliversun9 Dec 14, 2023
5f35c8d
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 14, 2023
7769f55
transform exclude paths
oliversun9 Dec 14, 2023
8dd13fa
fix
oliversun9 Dec 14, 2023
d0d07f1
produce minimal diff in lint and breaking rules
oliversun9 Dec 14, 2023
76ed0ac
remove TODOs on roots to excludes because they are already resolved
oliversun9 Dec 14, 2023
3f92634
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 15, 2023
0549c6d
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 15, 2023
0b56572
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 19, 2023
b839f52
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 20, 2023
cd9405a
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 21, 2023
290bbb8
post merge
oliversun9 Dec 21, 2023
942d5d8
commit
bufdev Dec 21, 2023
73a82d5
commit
bufdev Dec 21, 2023
0f1430b
Merge branch 'bufmod' into osun/migrate
oliversun9 Dec 21, 2023
62610fc
updates
oliversun9 Dec 22, 2023
9d9ed22
fix
oliversun9 Dec 22, 2023
1bbdd0d
merge
bufdev Dec 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 84 additions & 53 deletions private/buf/bufmigrate/bufmigrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,108 +26,139 @@ import (
"github.com/bufbuild/buf/private/pkg/storage/storageos"
)

// Migrate migrate buf configuration files.
// Migrate migrates buf configuration files.
//
// A buf.yaml v2 is written if any workspace directory or module directory is
// specified. The modules directories in the buf.yaml v2 will contain:
//
// - directories at moduleDirPaths
// - directories pointed to by buf.work.yamls at workspaceDirPaths
//
// More specifically:
//
// - If a workspace is specified, then all of its module directories are also migrated,
// regardless whether these module directories are specified in moduleDirPaths. Same
// behavior with multiple workspaces. For example, if workspace foo has directories
// bar and baz, then specifying foo, foo + bar and foo + bar + baz are the same.
// - If a workspace is specfied, and modules not from this workspace are specified, the
// buf.yaml will contain all directories from the workspace, as well as the module
// directories specified.
// - If only module directories are specified, then the buf.yaml will contain exactly
// these directories.
// - If a module specified is within some workspace not from workspaceDirPaths, we migrate
// the module directory only (updating/deciding on this behavior is still a TODO).
// - If only one workspace directory is specified and no module directory is specified,
// the buf.yaml will be written at <workspace directory>/buf.yaml. Otherwise, it will
// be written at ./buf.yaml.
//
// Each generation template will be overwritten by a file in v2.
func Migrate(
ctx context.Context,
clientProvider bufapi.ClientProvider,
messageWriter io.Writer,
storageProvider storageos.Provider,
clientProvider bufapi.ClientProvider,
workspaceDirPaths []string,
moduleDirPaths []string,
generateTemplatePaths []string,
options ...MigrateOption,
) (retErr error) {
migrateOptions := newMigrateOptions()
for _, option := range options {
option(migrateOptions)
}
if migrateOptions.bufWorkYAMLFilePath == "" && len(migrateOptions.bufYAMLFilePaths) == 0 {
return errors.New("unimplmented")
if len(workspaceDirPaths) == 0 && len(moduleDirPaths) == 0 && len(generateTemplatePaths) == 0 {
return errors.New("no directory or file specified")
}
// TODO: decide on behavior for where to write this file
destinationDir := "."
// Alternatively, we could do the following: (but "." is probably better, since we want users to
// have buf.yaml v2 at their repository root and they are likely running this command from there)
//
// if migrateOptions.bufWorkYAMLFilePath != "" {
// destinationDir = filepath.Base(migrateOptions.bufWorkYAMLFilePath)
// } else if len(migrateOptions.bufYAMLFilePaths) == 1 {
// // TODO: maybe use "." (or maybe add --dest flag)
// destinationDir = filepath.Base(migrateOptions.bufYAMLFilePaths[0])
// } else {
// destinationDir = "."
// }
var err error
// Directories cannot jump context because in the migrated buf.yaml v2, each
// directory path cannot jump context. I.e. it's not valid to have `- directory: ..`
// in a buf.yaml v2.
workspaceDirPaths, err = slicesext.MapError(
workspaceDirPaths,
func(workspaceDirPath string) (string, error) {
if !filepath.IsLocal(workspaceDirPath) {
return "", fmt.Errorf("%s is not a relative path", workspaceDirPath)
}
return filepath.Clean(workspaceDirPath), nil
},
)
if err != nil {
return err
}
moduleDirPaths, err = slicesext.MapError(
moduleDirPaths,
func(moduleDirPath string) (string, error) {
if !filepath.IsLocal(moduleDirPath) {
return "", fmt.Errorf("%s is not a relative path", moduleDirPath)
}
return filepath.Clean(moduleDirPath), nil
},
)
if err != nil {
return err
}
generateTemplatePaths = slicesext.Map(generateTemplatePaths, filepath.Clean)
bucket, err := storageProvider.NewReadWriteBucket(
".",
storageos.ReadWriteBucketWithSymlinksIfSupported(),
)
if err != nil {
return err
}
destionationDirectory := "."
if len(workspaceDirPaths) == 1 && len(moduleDirPaths) == 0 {
destionationDirectory = workspaceDirPaths[0]
}
migrator := newMigrator(
messageWriter,
clientProvider,
bucket,
destinationDir,
destionationDirectory,
)
if migrateOptions.bufWorkYAMLFilePath != "" {
for _, workspaceDirPath := range workspaceDirPaths {
if err := migrator.addWorkspaceDirectory(
ctx,
filepath.Dir(migrateOptions.bufWorkYAMLFilePath),
workspaceDirPath,
); err != nil {
return err
}
}
for _, bufYAMLPath := range migrateOptions.bufYAMLFilePaths {
// TODO: read upwards to make sure it's not in a workspace
for _, bufYAMLPath := range moduleDirPaths {
// TODO: read upwards to make sure it's not in a workspace.
// i.e. for ./foo/bar/buf.yaml, check none of "./foo", ".", "../", "../..", and etc. is a workspace.
// The logic for this is in getMapPathAndSubDirPath from buffetch/internal
if err := migrator.addModuleDirectory(
ctx,
filepath.Dir(bufYAMLPath),
bufYAMLPath,
); err != nil {
return err
}
}
for _, bufGenYAMLPath := range generateTemplatePaths {
if err := migrator.addBufGenYAML(bufGenYAMLPath); err != nil {
return err
}
}
if migrateOptions.dryRun {
return migrator.migrateAsDryRun(migrateOptions.dryRunWriter)
return migrator.migrateAsDryRun(ctx)
}
return migrator.migrate(ctx)
}

// MigrateOption is a migrate option.
type MigrateOption func(*migrateOptions)

// MigrateAsDryRun write to the writer the summary of the changes to be made, without writing to the disk.
func MigrateAsDryRun(writer io.Writer) MigrateOption {
// MigrateAsDryRun print the summary of the changes to be made, without writing to the disk.
func MigrateAsDryRun() MigrateOption {
return func(migrateOptions *migrateOptions) {
migrateOptions.dryRun = true
migrateOptions.dryRunWriter = writer
}
}

// MigrateBufWorkYAMLFile migrates a v1 buf.work.yaml.
func MigrateBufWorkYAMLFile(path string) (MigrateOption, error) {
// TODO: Looking at IsLocal's doc, it seems to validate for what we want: relative and does not jump context.
if !filepath.IsLocal(path) {
return nil, fmt.Errorf("%s is not a relative path", path)
}
return func(migrateOptions *migrateOptions) {
migrateOptions.bufWorkYAMLFilePath = filepath.Clean(path)
}, nil
}

// MigrateBufYAMLFile migrates buf.yaml files.
func MigrateBufYAMLFile(paths []string) (MigrateOption, error) {
for _, path := range paths {
if !filepath.IsLocal(path) {
return nil, fmt.Errorf("%s is not a relative path", path)
}
}
return func(migrateOptions *migrateOptions) {
migrateOptions.bufYAMLFilePaths = slicesext.Map(paths, filepath.Clean)
}, nil
}
/// *** PRIVATE ***

type migrateOptions struct {
dryRun bool
dryRunWriter io.Writer
bufWorkYAMLFilePath string
bufYAMLFilePaths []string
dryRun bool
}

func newMigrateOptions() *migrateOptions {
Expand Down
22 changes: 0 additions & 22 deletions private/buf/bufmigrate/bufmigrate_test.go

This file was deleted.

Loading
Loading