Skip to content

Commit

Permalink
commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@bufdev
bufdev committed Feb 29, 2024
1 parent 47d1e27 commit d90ae24
Show file tree
Hide file tree
Showing 10 changed files with 271 additions and 91 deletions.
2 changes: 1 addition & 1 deletion make/buf/all.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ godata: installspdx-go-data installwkt-go-data $(PROTOC)
rm -rf private/gen/data
mkdir -p private/gen/data/datawkt
mkdir -p private/gen/data/dataspdx
wkt-go-data $(CACHE_INCLUDE) --package datawkt > private/gen/data/datawkt/datawkt.gen.go
wkt-go-data "$(CACHE_INCLUDE)" --package datawkt --protobuf-version "$(PROTOC_VERSION)" > private/gen/data/datawkt/datawkt.gen.go
spdx-go-data --package dataspdx > private/gen/data/dataspdx/dataspdx.gen.go

prepostgenerate:: godata
Expand Down
55 changes: 39 additions & 16 deletions private/buf/bufctl/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil"
"github.com/bufbuild/buf/private/bufpkg/bufmodule"
"github.com/bufbuild/buf/private/bufpkg/bufreflect"
"github.com/bufbuild/buf/private/gen/data/datawkt"
imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1"
"github.com/bufbuild/buf/private/pkg/app"
"github.com/bufbuild/buf/private/pkg/app/appcmd"
Expand Down Expand Up @@ -91,11 +92,14 @@ type Controller interface {
input string,
options ...FunctionOption,
) ([]ImageWithConfig, error)
// GetImageFileInfos gets the .proto FileInfos for the given input.
// GetImportableImageFileInfos gets the importable .proto FileInfos for the given input.
//
// Imports are always included.
// Sorted by Path.
GetImageFileInfos(
// This includes all files that can be possible imported. For example, if a Module
// is given, this will return FileInfos for the Module, its dependencies, and all of
// the Well-Known Types.
//
// Returned ImageFileInfos are sorted by Path.
GetImportableImageFileInfos(
ctx context.Context,
input string,
options ...FunctionOption,
Expand Down Expand Up @@ -412,7 +416,7 @@ func (c *controller) GetTargetImageWithConfigs(
}
}

func (c *controller) GetImageFileInfos(
func (c *controller) GetImportableImageFileInfos(
ctx context.Context,
input string,
options ...FunctionOption,
Expand All @@ -431,46 +435,65 @@ func (c *controller) GetImageFileInfos(
if err != nil {
return nil, err
}
var imageFileInfos []bufimage.ImageFileInfo
switch t := ref.(type) {
case buffetch.ProtoFileRef:
workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions)
if err != nil {
return nil, err
}
return getImageFileInfosForModuleSet(ctx, workspace)
imageFileInfos, err = getImageFileInfosForModuleSet(ctx, workspace)
if err != nil {
return nil, err
}
case buffetch.SourceRef:
workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions)
if err != nil {
return nil, err
}
return getImageFileInfosForModuleSet(ctx, workspace)
imageFileInfos, err = getImageFileInfosForModuleSet(ctx, workspace)
if err != nil {
return nil, err
}
case buffetch.ModuleRef:
workspace, err := c.getWorkspaceForModuleRef(ctx, t, functionOptions)
if err != nil {
return nil, err
}
return getImageFileInfosForModuleSet(ctx, workspace)
imageFileInfos, err = getImageFileInfosForModuleSet(ctx, workspace)
if err != nil {
return nil, err
}
case buffetch.MessageRef:
image, err := c.getImageForMessageRef(ctx, t, functionOptions)
if err != nil {
return nil, err
}
imageFiles := image.Files()
imageFileInfos := make([]bufimage.ImageFileInfo, len(imageFiles))
imageFileInfos = make([]bufimage.ImageFileInfo, len(imageFiles))
for i, imageFile := range imageFiles {
imageFileInfos[i] = imageFile
}
sort.Slice(
imageFileInfos,
func(i int, j int) bool {
return imageFileInfos[i].Path() < imageFileInfos[j].Path()
},
)
return imageFileInfos, nil
default:
// This is a system error.
return nil, syserror.Newf("invalid Ref: %T", ref)
}
imageFileInfos, err = bufimage.AppendWellKnownTypeImageFileInfos(
ctx,
// TODO: Use a cached bucket backed on disk so we can get local paths.
datawkt.ReadBucket,
imageFileInfos,
)
if err != nil {
return nil, err
}
sort.Slice(
imageFileInfos,
func(i int, j int) bool {
return imageFileInfos[i].Path() < imageFileInfos[j].Path()
},
)
return imageFileInfos, nil
}

func (c *controller) PutImage(
Expand Down
33 changes: 26 additions & 7 deletions private/buf/bufwkt/cmd/wkt-go-data/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ import (
)

const (
programName = "wkt-go-data"
pkgFlagName = "package"
sliceLength = math.MaxInt64
programName = "wkt-go-data"
pkgFlagName = "package"
protobufVersionFlagName = "protobuf-version"
sliceLength = math.MaxInt64
)

var failedError = app.NewError( /* exitCode */ 100, "something went wrong")
Expand All @@ -69,7 +70,8 @@ func newCommand() *appcmd.Command {
}

type flags struct {
Pkg string
Pkg string
ProtobufVersion string
}

func newFlags() *flags {
Expand All @@ -83,6 +85,12 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) {
"",
"The name of the generated package.",
)
flagSet.StringVar(
&f.ProtobufVersion,
protobufVersionFlagName,
"",
"The version of protobuf used to get the Well-Known Types.",
)
}

func run(ctx context.Context, container appext.Container, flags *flags) error {
Expand All @@ -91,6 +99,10 @@ func run(ctx context.Context, container appext.Container, flags *flags) error {
if packageName == "" {
packageName = filepath.Base(dirPath)
}
protobufVersion := flags.ProtobufVersion
if protobufVersion == "" {
return appcmd.NewInvalidArgumentErrorf("--%s is required", protobufVersionFlagName)
}
readWriteBucket, err := storageos.NewProvider(storageos.ProviderWithSymlinks()).NewReadWriteBucket(dirPath)
if err != nil {
return err
Expand Down Expand Up @@ -127,6 +139,7 @@ func run(ctx context.Context, container appext.Container, flags *flags) error {
pathToImports[path] = imports
}
golangFileData, err := getGolangFileData(
protobufVersion,
pathToData,
fullNameToMessage,
fullNameToEnum,
Expand Down Expand Up @@ -204,6 +217,7 @@ func getProtosourceFiles(
}

func getGolangFileData(
protobufVersion string,
pathToData map[string][]byte,
fullNameToMessage map[string]bufprotosource.Message,
fullNameToEnum map[string]bufprotosource.Enum,
Expand Down Expand Up @@ -238,9 +252,14 @@ import (
"github.com/bufbuild/buf/private/pkg/storage"
"github.com/bufbuild/buf/private/pkg/storage/storagemem"
"github.com/bufbuild/buf/private/pkg/normalpath"
)
var (
)`)
p("\n\n")
p(`// Version is the version of github.com/protocolbuffers/protobuf used to extract the Well-Known Types.
const Version = "`)
p(protobufVersion)
p(`"`)
p("\n\n")
p(`var (
// ReadBucket is the storage.ReadBucket with the static data generated for this package.
ReadBucket storage.ReadBucket`)
p("\n\n")
Expand Down
107 changes: 74 additions & 33 deletions private/buf/cmd/buf/command/lsfiles/lsfiles.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"

"github.com/bufbuild/buf/private/buf/bufcli"
"github.com/bufbuild/buf/private/buf/bufctl"
"github.com/bufbuild/buf/private/bufpkg/bufanalysis"
"github.com/bufbuild/buf/private/bufpkg/bufimage"
"github.com/bufbuild/buf/private/gen/data/datawkt"
"github.com/bufbuild/buf/private/pkg/app/appcmd"
"github.com/bufbuild/buf/private/pkg/app/appext"
"github.com/bufbuild/buf/private/pkg/slicesext"
Expand All @@ -32,12 +34,13 @@ import (
)

const (
formatFlagName = "format"
configFlagName = "config"
errorFormatFlagName = "error-format"
includeImportsFlagName = "include-imports"
disableSymlinksFlagName = "disable-symlinks"
asImportPathsFlagName = "as-import-paths"
formatFlagName = "format"
configFlagName = "config"
errorFormatFlagName = "error-format"
includeImportsFlagName = "include-imports"
includeImportableFlagName = "include-importable"
disableSymlinksFlagName = "disable-symlinks"
asImportPathsFlagName = "as-import-paths"

formatText = "text"
formatJSON = "json"
Expand Down Expand Up @@ -69,11 +72,13 @@ func NewCommand(
}

type flags struct {
Format string
Config string
ErrorFormat string
IncludeImports bool
DisableSymlinks bool
Format string
Config string
IncludeImports bool
IncludeImportable bool
DisableSymlinks bool
// Deprecated. This flag no longer has any effect as we don't build images anymore.
ErrorFormat string
// Deprecated
AsImportPaths bool
// special
Expand Down Expand Up @@ -102,6 +107,21 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) {
strings.Join(allFormats, ", "),
),
)
flagSet.BoolVar(
&f.IncludeImports,
includeImportsFlagName,
false,
"Include imports",
)
flagSet.BoolVar(
&f.IncludeImportable,
includeImportableFlagName,
false,
fmt.Sprintf(
"Include all .proto file that are importable by the input. --%s is redundant if this is set",
includeImportsFlagName,
),
)
flagSet.StringVar(
&f.ErrorFormat,
errorFormatFlagName,
Expand All @@ -111,12 +131,8 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) {
stringutil.SliceToString(bufanalysis.AllFormatStrings),
),
)
flagSet.BoolVar(
&f.IncludeImports,
includeImportsFlagName,
false,
"Include imports",
)
_ = flagSet.MarkDeprecated(errorFormatFlagName, "This flag no longer has any effect")
_ = flagSet.MarkHidden(errorFormatFlagName)
flagSet.BoolVar(
&f.AsImportPaths,
asImportPathsFlagName,
Expand All @@ -142,41 +158,64 @@ func run(
controller, err := bufcli.NewController(
container,
bufctl.WithDisableSymlinks(flags.DisableSymlinks),
bufctl.WithFileAnnotationErrorFormat(flags.ErrorFormat),
)
if err != nil {
return err
}
// Sorted.
imageFileInfos, err := controller.GetImageFileInfos(
imageFileInfos, err := controller.GetImportableImageFileInfos(
ctx,
input,
bufctl.WithConfigOverride(flags.Config),
)
if err != nil {
return err
}
if !flags.IncludeImports {
imageFileInfos = slicesext.Filter(
imageFileInfos,
func(imageFileInfo bufimage.ImageFileInfo) bool {
return !imageFileInfo.IsImport()
},
)
} else {
// Also automatically adds imported WKTs if not present.
imageFileInfos, err = bufimage.ImageFileInfosWithOnlyTargetsAndTargetImports(imageFileInfos)
if err != nil {
return err
if !flags.IncludeImportable {
if !flags.IncludeImports {
imageFileInfos = slicesext.Filter(
imageFileInfos,
func(imageFileInfo bufimage.ImageFileInfo) bool {
return !imageFileInfo.IsImport()
},
)
} else {
// Also automatically adds imported WKTs if not present.
imageFileInfos, err = bufimage.ImageFileInfosWithOnlyTargetsAndTargetImports(
ctx,
datawkt.ReadBucket,
imageFileInfos,
)
if err != nil {
return err
}
}
}
var formatFunc func(bufimage.ImageFileInfo) (string, error)
switch flags.Format {
case formatText:
sort.Slice(
imageFileInfos,
func(i int, j int) bool {
return imageFileInfos[i].ExternalPath() < imageFileInfos[j].ExternalPath()
},
)
formatFunc = func(imageFileInfo bufimage.ImageFileInfo) (string, error) {
return imageFileInfo.ExternalPath(), nil
}
case formatJSON:
sort.Slice(
imageFileInfos,
func(i int, j int) bool {
if imageFileInfos[i].LocalPath() < imageFileInfos[j].LocalPath() {
return true
}
if imageFileInfos[i].LocalPath() > imageFileInfos[j].LocalPath() {
return false
}
return imageFileInfos[i].Path() < imageFileInfos[j].Path()
},
)
formatFunc = func(imageFileInfo bufimage.ImageFileInfo) (string, error) {
data, err := json.Marshal(newExternalImageFileInfo(imageFileInfo))
if err != nil {
Expand Down Expand Up @@ -204,8 +243,8 @@ func run(
}

type externalImageFileInfo struct {
Path string `json:"path" yaml:"path"`
ImportPath string `json:"import_path" yaml:"import_path"`
LocalPath string `json:"local_path" yaml:"local_path"`
Module string `json:"module" yaml:"module"`
Commit string `json:"commit" yaml:"commit"`
IsImport bool `json:"is_import" yaml:"is_import"`
Expand All @@ -221,8 +260,10 @@ func newExternalImageFileInfo(imageFileInfo bufimage.ImageFileInfo) *externalIma
commit = commitID.String()
}
return &externalImageFileInfo{
Path: imageFileInfo.LocalPath(),
// This seems backwards when you read it, but it is right: the Path is the import path,
// the LocalPath is the path that a user would have for a file on disk.
ImportPath: imageFileInfo.Path(),
LocalPath: imageFileInfo.LocalPath(),
Module: module,
Commit: commit,
IsImport: imageFileInfo.IsImport(),
Expand Down
Loading

0 comments on commit d90ae24

Please sign in to comment.