diff --git a/private/buf/bufcli/bufcli.go b/private/buf/bufcli/bufcli.go index 645e1b8766..6ea87f242b 100644 --- a/private/buf/bufcli/bufcli.go +++ b/private/buf/bufcli/bufcli.go @@ -487,7 +487,7 @@ func NewWireImageReader( ) bufwire.ImageReader { return bufwire.NewImageReader( logger, - newFetchImageReader(logger, storageosProvider, runner), + newFetchMessageReader(logger, storageosProvider, runner), ) } @@ -506,9 +506,12 @@ func NewWireImageWriter( // NewWireProtoEncodingReader returns a new ProtoEncodingReader. func NewWireProtoEncodingReader( logger *zap.Logger, + storageosProvider storageos.Provider, + runner command.Runner, ) bufwire.ProtoEncodingReader { return bufwire.NewProtoEncodingReader( logger, + newFetchMessageReader(logger, storageosProvider, runner), ) } @@ -518,6 +521,9 @@ func NewWireProtoEncodingWriter( ) bufwire.ProtoEncodingWriter { return bufwire.NewProtoEncodingWriter( logger, + buffetch.NewWriter( + logger, + ), ) } @@ -1010,14 +1016,14 @@ func newFetchSourceReader( ) } -// newFetchImageReader creates a new buffetch.ImageReader with the default HTTP client +// newFetchMessageReader creates a new buffetch.MessageReader with the default HTTP client // and git cloner. -func newFetchImageReader( +func newFetchMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, runner command.Runner, -) buffetch.ImageReader { - return buffetch.NewImageReader( +) buffetch.MessageReader { + return buffetch.NewMessageReader( logger, storageosProvider, defaultHTTPClient, diff --git a/private/buf/bufconvert/bufconvert.go b/private/buf/bufconvert/bufconvert.go deleted file mode 100644 index 67267c6473..0000000000 --- a/private/buf/bufconvert/bufconvert.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconvert - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - "github.com/bufbuild/buf/private/buf/bufref" - "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/codes" -) - -const ( - // MessageEncodingBinpb is the binary image encoding. - MessageEncodingBinpb MessageEncoding = iota + 1 - // MessageEncodingJSON is the JSON image encoding. - MessageEncodingJSON - // MessageEncodingTxtpb is the protobuf text image encoding. - MessageEncodingTxtpb - // MessageEncodingYAML is the YAML image encoding. - MessageEncodingYAML - // formatBinpb is the binary format. - formatBinpb = "binpb" - // formatJSON is the JSON format. - formatJSON = "json" - // formatTxtpb is the protobuf text format. - formatTxtpb = "txtpb" - // formatYAML is the YAML format. - formatYAML = "yaml" - - // formatBin is the binary format's old form, now deprecated. - formatBin = "bin" -) - -var ( - // MessageEncodingFormatsString is the string representation of all message encoding formats. - // - // This does not include deprecated formats. - MessageEncodingFormatsString = stringutil.SliceToString(messageEncodingFormats) - // sorted - messageEncodingFormats = []string{ - formatBinpb, - formatJSON, - formatTxtpb, - formatYAML, - } -) - -// MessageEncoding is the encoding of the message -type MessageEncoding int - -// MessageEncodingRef is a message encoding file reference. -type MessageEncodingRef interface { - Path() string - MessageEncoding() MessageEncoding -} - -// NewMessageEncodingRef returns a new MessageEncodingRef. -func NewMessageEncodingRef( - ctx context.Context, - value string, - defaultEncoding MessageEncoding, -) (MessageEncodingRef, error) { - ctx, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "new_message_encoding_ref") - defer span.End() - path, messageEncoding, err := getPathAndMessageEncoding(ctx, value, defaultEncoding) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return nil, err - } - return newMessageEncodingRef(path, messageEncoding), nil -} - -func getPathAndMessageEncoding( - ctx context.Context, - value string, - defaultEncoding MessageEncoding, -) (string, MessageEncoding, error) { - path, options, err := bufref.GetRawPathAndOptions(value) - if err != nil { - return "", 0, err - } - messageEncoding := parseMessageEncodingExt(filepath.Ext(path), defaultEncoding) - for key, value := range options { - switch key { - case "format": - if app.IsDevNull(path) { - return "", 0, fmt.Errorf("not allowed if path is %s", app.DevNullFilePath) - } - messageEncoding, err = parseMessageEncodingFormat(value) - if err != nil { - return "", 0, err - } - default: - return "", 0, fmt.Errorf("invalid options key: %q", key) - } - } - return path, messageEncoding, nil -} - -func parseMessageEncodingExt(ext string, defaultEncoding MessageEncoding) MessageEncoding { - switch strings.TrimPrefix(ext, ".") { - case formatBin, formatBinpb: - return MessageEncodingBinpb - case formatJSON: - return MessageEncodingJSON - case formatTxtpb: - return MessageEncodingTxtpb - case formatYAML: - return MessageEncodingYAML - default: - return defaultEncoding - } -} - -func parseMessageEncodingFormat(format string) (MessageEncoding, error) { - switch format { - case formatBin, formatBinpb: - return MessageEncodingBinpb, nil - case formatJSON: - return MessageEncodingJSON, nil - case formatTxtpb: - return MessageEncodingTxtpb, nil - case formatYAML: - return MessageEncodingYAML, nil - default: - return 0, fmt.Errorf("invalid format for message: %q", format) - } -} diff --git a/private/buf/bufconvert/message_encoding_ref.go b/private/buf/bufconvert/message_encoding_ref.go deleted file mode 100644 index 3790ba75e3..0000000000 --- a/private/buf/bufconvert/message_encoding_ref.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconvert - -var _ MessageEncodingRef = &messageEncodingRef{} - -type messageEncodingRef struct { - path string - messageEncoding MessageEncoding -} - -func newMessageEncodingRef( - path string, - messageEncoding MessageEncoding, -) *messageEncodingRef { - return &messageEncodingRef{ - path: path, - messageEncoding: messageEncoding, - } -} - -func (r *messageEncodingRef) Path() string { - return r.path -} - -func (r *messageEncodingRef) MessageEncoding() MessageEncoding { - return r.messageEncoding -} diff --git a/private/buf/bufconvert/usage.gen.go b/private/buf/bufconvert/usage.gen.go deleted file mode 100644 index 29527dd3b1..0000000000 --- a/private/buf/bufconvert/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufconvert - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 2b92b24157..e30a939852 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -30,21 +30,24 @@ import ( ) const ( - // ImageEncodingBin is the binary image encoding. - ImageEncodingBin ImageEncoding = iota + 1 - // ImageEncodingJSON is the JSON image encoding. - ImageEncodingJSON - // ImageEncodingTxtpb is the text protobuf image encoding. - ImageEncodingTxtpb - // ImageEncodingYAML is the YAML image encoding. - ImageEncodingYAML + // MessageEncodingBinpb is the binary message encoding. + MessageEncodingBinpb MessageEncoding = iota + 1 + // MessageEncodingJSON is the JSON message encoding. + MessageEncodingJSON + // MessageEncodingTxtpb is the text protobuf message encoding. + MessageEncodingTxtpb + // MessageEncodingYAML is the YAML message encoding. + MessageEncodingYAML + + useProtoNamesKey = "use_proto_names" + useEnumNumbersKey = "use_enum_numbers" ) var ( - // ImageFormatsString is the string representation of all image formats. + // MessageFormatsString is the string representation of all message formats. // // This does not include deprecated formats. - ImageFormatsString = stringutil.SliceToString(imageFormatsNotDeprecated) + MessageFormatsString = stringutil.SliceToString(messageFormatsNotDeprecated) // SourceDirFormatsString is the string representation of all source directory formats. // This includes all of the formats in SourceFormatsString except the protofile format. // @@ -70,8 +73,8 @@ var ( AllFormatsString = stringutil.SliceToString(allFormatsNotDeprecated) ) -// ImageEncoding is the encoding of the image. -type ImageEncoding int +// MessageEncoding is the encoding of the message. +type MessageEncoding int // PathResolver resolves external paths to paths. type PathResolver interface { @@ -92,19 +95,27 @@ type PathResolver interface { PathForExternalPath(externalPath string) (string, error) } -// Ref is an image file or source bucket reference. +// Ref is an message file or source bucket reference. type Ref interface { PathResolver internalRef() internal.Ref } -// ImageRef is an image file reference. -type ImageRef interface { +// MessageRef is an message file reference. +type MessageRef interface { Ref - ImageEncoding() ImageEncoding + MessageEncoding() MessageEncoding + // Path returns the path of the file. + // + // May be used for items such as YAML unmarshaling errors. + Path() string + // UseProtoNames only applies for MessageEncodingYAML at this time. + UseProtoNames() bool + // UseEnumNumbers only applies for MessageEncodingYAML at this time. + UseEnumNumbers() bool IsNull() bool - internalFileRef() internal.FileRef + internalSingleRef() internal.SingleRef } // SourceOrModuleRef is a source bucket or module reference. @@ -132,10 +143,10 @@ type ProtoFileRef interface { internalProtoFileRef() internal.ProtoFileRef } -// ImageRefParser is an image ref parser for Buf. -type ImageRefParser interface { - // GetImageRef gets the reference for the image file. - GetImageRef(ctx context.Context, value string) (ImageRef, error) +// MessageRefParser is an message ref parser for Buf. +type MessageRefParser interface { + // GetMessageRef gets the reference for the message file. + GetMessageRef(ctx context.Context, value string) (MessageRef, error) } // SourceRefParser is a source ref parser for Buf. @@ -157,16 +168,16 @@ type SourceOrModuleRefParser interface { SourceRefParser ModuleRefParser - // GetSourceOrModuleRef gets the reference for the image file or source bucket. + // GetSourceOrModuleRef gets the reference for the message file or source bucket. GetSourceOrModuleRef(ctx context.Context, value string) (SourceOrModuleRef, error) } // RefParser is a ref parser for Buf. type RefParser interface { - ImageRefParser + MessageRefParser SourceOrModuleRefParser - // GetRef gets the reference for the image file, source bucket, or module. + // GetRef gets the reference for the message file, source bucket, or module. GetRef(ctx context.Context, value string) (Ref, error) } @@ -177,11 +188,21 @@ func NewRefParser(logger *zap.Logger) RefParser { return newRefParser(logger) } -// NewImageRefParser returns a new RefParser for images only. +// NewMessageRefParser returns a new RefParser for messages only. +func NewMessageRefParser(logger *zap.Logger, options ...MessageRefParserOption) MessageRefParser { + return newMessageRefParser(logger, options...) +} + +// MessageRefParserOption is an option for a new MessageRefParser. +type MessageRefParserOption func(*messageRefParserOptions) + +// MessageRefParserWithDefaultMessageEncoding says to use the default MessageEncoding. // -// This defaults to binary. -func NewImageRefParser(logger *zap.Logger) ImageRefParser { - return newImageRefParser(logger) +// The default default is MessageEncodingBinpb. +func MessageRefParserWithDefaultMessageEncoding(defaultMessageEncoding MessageEncoding) MessageRefParserOption { + return func(messageRefParserOptions *messageRefParserOptions) { + messageRefParserOptions.defaultMessageEncoding = defaultMessageEncoding + } } // NewSourceRefParser returns a new RefParser for sources only. @@ -218,15 +239,15 @@ type ReadWriteBucketCloser internal.ReadWriteBucketCloser // ReadBucketCloserWithTerminateFileProvider is a ReadBucketCloser with a TerminateFileProvider. type ReadBucketCloserWithTerminateFileProvider internal.ReadBucketCloserWithTerminateFileProvider -// ImageReader is an image reader. -type ImageReader interface { - // GetImageFile gets the image file. +// MessageReader is an message reader. +type MessageReader interface { + // GetMessageFile gets the message file. // // The returned file will be uncompressed. - GetImageFile( + GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.ReadCloser, error) } @@ -267,7 +288,7 @@ type ModuleFetcher interface { // Reader is a reader for Buf. type Reader interface { - ImageReader + MessageReader SourceReader ModuleFetcher } @@ -293,15 +314,15 @@ func NewReader( ) } -// NewImageReader returns a new ImageReader. -func NewImageReader( +// NewMessageReader returns a new MessageReader. +func NewMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, httpClient *http.Client, httpAuthenticator httpauth.Authenticator, gitCloner git.Cloner, -) ImageReader { - return newImageReader( +) MessageReader { + return newMessageReader( logger, storageosProvider, httpClient, @@ -344,11 +365,11 @@ func NewModuleFetcher( // Writer is a writer for Buf. type Writer interface { - // PutImageFile puts the image file. - PutImageFile( + // PutMessageFile puts the message file. + PutMessageFile( ctx context.Context, container app.EnvStdoutContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.WriteCloser, error) } diff --git a/private/buf/buffetch/format.go b/private/buf/buffetch/format.go index 6d704f28e4..7ea7bcd0e9 100644 --- a/private/buf/buffetch/format.go +++ b/private/buf/buffetch/format.go @@ -48,7 +48,7 @@ const ( var ( // sorted - imageFormats = []string{ + messageFormats = []string{ formatBin, formatBinpb, formatBingz, @@ -58,7 +58,7 @@ var ( formatYAML, } // sorted - imageFormatsNotDeprecated = []string{ + messageFormatsNotDeprecated = []string{ formatBinpb, formatJSON, formatTxtpb, @@ -150,4 +150,11 @@ var ( formatJSONGZ: formatJSON, formatTargz: formatTar, } + + messageEncodingToFormat = map[MessageEncoding]string{ + MessageEncodingBinpb: formatBinpb, + MessageEncodingJSON: formatJSON, + MessageEncodingTxtpb: formatTxtpb, + MessageEncodingYAML: formatYAML, + } ) diff --git a/private/buf/buffetch/image_ref.go b/private/buf/buffetch/image_ref.go deleted file mode 100644 index b9eb995228..0000000000 --- a/private/buf/buffetch/image_ref.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffetch - -import ( - "github.com/bufbuild/buf/private/buf/buffetch/internal" - "github.com/bufbuild/buf/private/pkg/normalpath" -) - -var _ ImageRef = &imageRef{} - -type imageRef struct { - fileRef internal.FileRef - imageEncoding ImageEncoding -} - -func newImageRef( - fileRef internal.FileRef, - imageEncoding ImageEncoding, -) *imageRef { - return &imageRef{ - fileRef: fileRef, - imageEncoding: imageEncoding, - } -} - -func (r *imageRef) PathForExternalPath(externalPath string) (string, error) { - return normalpath.NormalizeAndValidate(externalPath) -} - -func (r *imageRef) ImageEncoding() ImageEncoding { - return r.imageEncoding -} - -func (r *imageRef) IsNull() bool { - return r.fileRef.FileScheme() == internal.FileSchemeNull -} - -func (r *imageRef) internalRef() internal.Ref { - return r.fileRef -} - -func (r *imageRef) internalFileRef() internal.FileRef { - return r.fileRef -} diff --git a/private/buf/buffetch/internal/archive_ref.go b/private/buf/buffetch/internal/archive_ref.go index 09552f0289..9ca53be8ad 100644 --- a/private/buf/buffetch/internal/archive_ref.go +++ b/private/buf/buffetch/internal/archive_ref.go @@ -45,6 +45,7 @@ func newArchiveRef( format, path, compressionType, + nil, ) if err != nil { return nil, err diff --git a/private/buf/buffetch/internal/errors.go b/private/buf/buffetch/internal/errors.go index 1c9cd4ac13..bcc11d8da7 100644 --- a/private/buf/buffetch/internal/errors.go +++ b/private/buf/buffetch/internal/errors.go @@ -78,9 +78,17 @@ func NewNoPathError() error { return errors.New("value has no path once processed") } -// NewOptionsInvalidKeyError is a fetch error. -func NewOptionsInvalidKeyError(key string) error { - return fmt.Errorf("invalid options key: %q", key) +// NewOptionsInvalidKeysError is a fetch error. +func NewOptionsInvalidKeysError(keys ...string) error { + if len(keys) == 1 { + return fmt.Errorf("invalid key: %q", keys[0]) + } + return fmt.Errorf("invalid keys: \"%v\"", strings.Join(keys, ", ")) +} + +// NewOptionsInvalidValueForKeyError is a fetch error. +func NewOptionsInvalidValueForKeyError(key string, value string) error { + return fmt.Errorf("invalid value %q for key: %q", value, key) } // NewOptionsInvalidForFormatError is a fetch error. @@ -178,3 +186,27 @@ func NewWriteLocalDisabledError() error { func NewWriteStdioDisabledError() error { return NewWriteDisabledError("stdout") } + +func newValueEmptyError() error { + return errors.New("required") +} + +func newValueMultipleHashtagsError(value string) error { + return fmt.Errorf("%q has multiple #s which is invalid", value) +} + +func newValueStartsWithHashtagError(value string) error { + return fmt.Errorf("%q starts with # which is invalid", value) +} + +func newValueEndsWithHashtagError(value string) error { + return fmt.Errorf("%q ends with # which is invalid", value) +} + +func newOptionsInvalidError(s string) error { + return fmt.Errorf("invalid options: %q", s) +} + +func newOptionsDuplicateKeyError(key string) error { + return fmt.Errorf("duplicate options key: %q", key) +} diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 5983dc7658..6e43764efd 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -112,12 +112,13 @@ type BucketRef interface { // SingleRef is a non-archive file reference. type SingleRef interface { FileRef + CustomOptionValue(key string) (string, bool) singleRef() } // NewSingleRef returns a new SingleRef. func NewSingleRef(path string, compressionType CompressionType) (SingleRef, error) { - return newSingleRef("", path, compressionType) + return newSingleRef("", path, compressionType, nil) } // ArchiveRef is an archive reference. @@ -252,12 +253,14 @@ func NewDirectParsedSingleRef( path string, fileScheme FileScheme, compressionType CompressionType, + customOptions map[string]string, ) ParsedSingleRef { return newDirectSingleRef( format, path, fileScheme, compressionType, + customOptions, ) } @@ -528,9 +531,13 @@ type RawRef struct { ArchiveStripComponents uint32 // Only set for proto file ref format. // Sets whether or not to include the files in the rest of the package - // in the image for the ProtoFileRef. + // in the message for the ProtoFileRef. // This defaults to false. IncludePackageFiles bool + // Any unrecognized options. Some formats may allow custom options, and those + // formats should check for those custom options in this map. If a format + // does not allow an option, an error will be returned. + UnrecognizedOptions map[string]string } // RefParserOption is an RefParser option. @@ -658,6 +665,13 @@ func WithSingleDefaultCompressionType(defaultCompressionType CompressionType) Si } } +// WithSingleCustomOptionKey adds a custom option key that is recognized.. +func WithSingleCustomOptionKey(key string) SingleFormatOption { + return func(singleFormatInfo *singleFormatInfo) { + singleFormatInfo.customOptionKeys[key] = struct{}{} + } +} + // ArchiveFormatOption is a archive format option. type ArchiveFormatOption func(*archiveFormatInfo) diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index 49071108c4..e88c61a15a 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -16,9 +16,10 @@ package internal import ( "context" + "sort" "strconv" + "strings" - "github.com/bufbuild/buf/private/buf/bufref" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/git" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -87,8 +88,19 @@ func (a *refParser) getParsedRef( return nil, NewFormatNotAllowedError(rawRef.Format, allowedFormats) } } + if !singleOK && len(rawRef.UnrecognizedOptions) > 0 { + // Only single refs allow custom options. In every other case, this is an error. + // + // We verify unrecognized options match what is expected in getSingleRef. + keys := make([]string, 0, len(rawRef.UnrecognizedOptions)) + for key := range rawRef.UnrecognizedOptions { + keys = append(keys, key) + } + sort.Strings(keys) + return nil, NewOptionsInvalidKeysError(keys...) + } if singleOK { - return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType) + return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType, singleFormatInfo.customOptionKeys) } if archiveOK { return getArchiveRef(rawRef, archiveFormatInfo.archiveType, archiveFormatInfo.defaultCompressionType) @@ -111,12 +123,13 @@ func (a *refParser) getParsedRef( // validated per rules on rawRef func (a *refParser) getRawRef(value string) (*RawRef, error) { // path is never empty after returning from this function - path, options, err := bufref.GetRawPathAndOptions(value) + path, options, err := getRawPathAndOptions(value) if err != nil { return nil, err } rawRef := &RawRef{ - Path: path, + Path: path, + UnrecognizedOptions: make(map[string]string), } if a.rawRefProcessor != nil { if err := a.rawRefProcessor(rawRef); err != nil { @@ -195,10 +208,10 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { case "false", "": rawRef.IncludePackageFiles = false default: - return nil, NewOptionsInvalidKeyError(key) + return nil, NewOptionsInvalidValueForKeyError(key, value) } default: - return nil, NewOptionsInvalidKeyError(key) + rawRef.UnrecognizedOptions[key] = value } } @@ -249,18 +262,71 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { return rawRef, nil } +// getRawPathAndOptions returns the raw path and options from the value provided, +// the rawPath will be non-empty when returning without error here. +func getRawPathAndOptions(value string) (string, map[string]string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", nil, newValueEmptyError() + } + + switch splitValue := strings.Split(value, "#"); len(splitValue) { + case 1: + return value, nil, nil + case 2: + path := strings.TrimSpace(splitValue[0]) + optionsString := strings.TrimSpace(splitValue[1]) + if path == "" { + return "", nil, newValueStartsWithHashtagError(value) + } + if optionsString == "" { + return "", nil, newValueEndsWithHashtagError(value) + } + options := make(map[string]string) + for _, pair := range strings.Split(optionsString, ",") { + split := strings.Split(pair, "=") + if len(split) != 2 { + return "", nil, newOptionsInvalidError(optionsString) + } + key := strings.TrimSpace(split[0]) + value := strings.TrimSpace(split[1]) + if key == "" || value == "" { + return "", nil, newOptionsInvalidError(optionsString) + } + if _, ok := options[key]; ok { + return "", nil, newOptionsDuplicateKeyError(key) + } + options[key] = value + } + return path, options, nil + default: + return "", nil, newValueMultipleHashtagsError(value) + } +} + func getSingleRef( rawRef *RawRef, defaultCompressionType CompressionType, + customOptionKeys map[string]struct{}, ) (ParsedSingleRef, error) { compressionType := rawRef.CompressionType if compressionType == 0 { compressionType = defaultCompressionType } + var invalidKeys []string + for key := range rawRef.UnrecognizedOptions { + if _, ok := customOptionKeys[key]; !ok { + invalidKeys = append(invalidKeys, key) + } + } + if len(invalidKeys) > 0 { + return nil, NewOptionsInvalidKeysError(invalidKeys...) + } return newSingleRef( rawRef.Format, rawRef.Path, compressionType, + rawRef.UnrecognizedOptions, ) } @@ -354,11 +420,13 @@ func getProtoFileRef(rawRef *RawRef) (ParsedProtoFileRef, error) { type singleFormatInfo struct { defaultCompressionType CompressionType + customOptionKeys map[string]struct{} } func newSingleFormatInfo() *singleFormatInfo { return &singleFormatInfo{ defaultCompressionType: CompressionTypeNone, + customOptionKeys: make(map[string]struct{}), } } diff --git a/private/buf/bufref/bufref_test.go b/private/buf/buffetch/internal/ref_parser_test.go similarity index 97% rename from private/buf/bufref/bufref_test.go rename to private/buf/buffetch/internal/ref_parser_test.go index 9fe16ddfbb..83fbd6a721 100644 --- a/private/buf/bufref/bufref_test.go +++ b/private/buf/buffetch/internal/ref_parser_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bufref +package internal import ( "testing" @@ -81,7 +81,7 @@ func testGetRawPathAndOptionsError( ) { t.Run(value, func(t *testing.T) { t.Parallel() - _, _, err := GetRawPathAndOptions(value) + _, _, err := getRawPathAndOptions(value) assert.EqualError(t, err, expectedErr.Error()) }) } diff --git a/private/buf/buffetch/internal/single_ref.go b/private/buf/buffetch/internal/single_ref.go index addd35333c..1fc1ddebd1 100644 --- a/private/buf/buffetch/internal/single_ref.go +++ b/private/buf/buffetch/internal/single_ref.go @@ -36,12 +36,14 @@ type singleRef struct { path string fileScheme FileScheme compressionType CompressionType + customOptions map[string]string } func newSingleRef( format string, path string, compressionType CompressionType, + customOptions map[string]string, ) (*singleRef, error) { if path == "" { return nil, NewNoPathError() @@ -55,6 +57,7 @@ func newSingleRef( "", FileSchemeStdio, compressionType, + customOptions, ), nil } if app.IsDevStdin(path) { @@ -63,6 +66,7 @@ func newSingleRef( "", FileSchemeStdin, compressionType, + customOptions, ), nil } if app.IsDevStdout(path) { @@ -71,6 +75,7 @@ func newSingleRef( "", FileSchemeStdout, compressionType, + customOptions, ), nil } if app.IsDevNull(path) { @@ -79,6 +84,7 @@ func newSingleRef( "", FileSchemeNull, compressionType, + customOptions, ), nil } for prefix, fileScheme := range fileSchemePrefixToFileScheme { @@ -95,6 +101,7 @@ func newSingleRef( path, fileScheme, compressionType, + customOptions, ), nil } } @@ -106,6 +113,7 @@ func newSingleRef( normalpath.Normalize(path), FileSchemeLocal, compressionType, + customOptions, ), nil } @@ -114,12 +122,17 @@ func newDirectSingleRef( path string, fileScheme FileScheme, compressionType CompressionType, + customOptions map[string]string, ) *singleRef { + if customOptions == nil { + customOptions = make(map[string]string) + } return &singleRef{ format: format, path: path, fileScheme: fileScheme, compressionType: compressionType, + customOptions: customOptions, } } @@ -139,6 +152,11 @@ func (r *singleRef) CompressionType() CompressionType { return r.compressionType } +func (r *singleRef) CustomOptionValue(key string) (string, bool) { + value, ok := r.customOptions[key] + return value, ok +} + func (*singleRef) ref() {} func (*singleRef) fileRef() {} func (*singleRef) singleRef() {} diff --git a/private/buf/buffetch/message_ref.go b/private/buf/buffetch/message_ref.go new file mode 100644 index 0000000000..ce3a2d16d2 --- /dev/null +++ b/private/buf/buffetch/message_ref.go @@ -0,0 +1,96 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffetch + +import ( + "github.com/bufbuild/buf/private/buf/buffetch/internal" + "github.com/bufbuild/buf/private/pkg/normalpath" +) + +var _ MessageRef = &messageRef{} + +type messageRef struct { + singleRef internal.SingleRef + useProtoNames bool + useEnumNumbers bool + messageEncoding MessageEncoding +} + +func newMessageRef( + singleRef internal.SingleRef, + messageEncoding MessageEncoding, +) (*messageRef, error) { + useProtoNames, err := getTrueOrFalseForSingleRef(singleRef, useProtoNamesKey) + if err != nil { + return nil, err + } + useEnumNumbers, err := getTrueOrFalseForSingleRef(singleRef, useEnumNumbersKey) + if err != nil { + return nil, err + } + return &messageRef{ + singleRef: singleRef, + useProtoNames: useProtoNames, + useEnumNumbers: useEnumNumbers, + messageEncoding: messageEncoding, + }, nil +} + +func (r *messageRef) PathForExternalPath(externalPath string) (string, error) { + return normalpath.NormalizeAndValidate(externalPath) +} + +func (r *messageRef) MessageEncoding() MessageEncoding { + return r.messageEncoding +} + +func (r *messageRef) Path() string { + return r.singleRef.Path() +} + +func (r *messageRef) UseProtoNames() bool { + return r.useProtoNames +} + +func (r *messageRef) UseEnumNumbers() bool { + return r.useEnumNumbers +} + +func (r *messageRef) IsNull() bool { + return r.singleRef.FileScheme() == internal.FileSchemeNull +} + +func (r *messageRef) internalRef() internal.Ref { + return r.singleRef +} + +func (r *messageRef) internalSingleRef() internal.SingleRef { + return r.singleRef +} + +func getTrueOrFalseForSingleRef(singleRef internal.SingleRef, key string) (bool, error) { + value, ok := singleRef.CustomOptionValue(key) + if !ok { + return false, nil + } + switch value { + case "true": + return true, nil + case "false": + return false, nil + default: + return false, internal.NewOptionsInvalidValueForKeyError(key, value) + } +} diff --git a/private/buf/buffetch/reader.go b/private/buf/buffetch/reader.go index 8194da79c2..a18b7f5b9c 100644 --- a/private/buf/buffetch/reader.go +++ b/private/buf/buffetch/reader.go @@ -64,7 +64,7 @@ func newReader( } } -func newImageReader( +func newMessageReader( logger *zap.Logger, storageosProvider storageos.Provider, httpClient *http.Client, @@ -127,12 +127,12 @@ func newModuleFetcher( } } -func (a *reader) GetImageFile( +func (a *reader) GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.ReadCloser, error) { - return a.internalReader.GetFile(ctx, container, imageRef.internalFileRef()) + return a.internalReader.GetFile(ctx, container, messageRef.internalSingleRef()) } func (a *reader) GetSourceBucket( diff --git a/private/buf/buffetch/ref_parser.go b/private/buf/buffetch/ref_parser.go index bf86b0580b..a252fca81f 100644 --- a/private/buf/buffetch/ref_parser.go +++ b/private/buf/buffetch/ref_parser.go @@ -48,12 +48,20 @@ func newRefParser(logger *zap.Logger) *refParser { tracer: otel.GetTracerProvider().Tracer(tracerName), fetchRefParser: internal.NewRefParser( logger, - internal.WithRawRefProcessor(newRawRefProcessor()), + internal.WithRawRefProcessor(processRawRef), internal.WithSingleFormat(formatBin), internal.WithSingleFormat(formatBinpb), - internal.WithSingleFormat(formatJSON), + internal.WithSingleFormat( + formatJSON, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat(formatTxtpb), - internal.WithSingleFormat(formatYAML), + internal.WithSingleFormat( + formatYAML, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat( formatBingz, internal.WithSingleDefaultCompressionType( @@ -89,17 +97,29 @@ func newRefParser(logger *zap.Logger) *refParser { } } -func newImageRefParser(logger *zap.Logger) *refParser { +func newMessageRefParser(logger *zap.Logger, options ...MessageRefParserOption) *refParser { + messageRefParserOptions := newMessageRefParserOptions() + for _, option := range options { + option(messageRefParserOptions) + } return &refParser{ logger: logger.Named(loggerName), fetchRefParser: internal.NewRefParser( logger, - internal.WithRawRefProcessor(processRawRefImage), + internal.WithRawRefProcessor(newProcessRawRefMessage(messageRefParserOptions.defaultMessageEncoding)), internal.WithSingleFormat(formatBin), internal.WithSingleFormat(formatBinpb), - internal.WithSingleFormat(formatJSON), + internal.WithSingleFormat( + formatJSON, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat(formatTxtpb), - internal.WithSingleFormat(formatYAML), + internal.WithSingleFormat( + formatYAML, + internal.WithSingleCustomOptionKey(useProtoNamesKey), + internal.WithSingleCustomOptionKey(useEnumNumbersKey), + ), internal.WithSingleFormat( formatBingz, internal.WithSingleDefaultCompressionType( @@ -204,11 +224,11 @@ func (a *refParser) GetRef( } switch t := parsedRef.(type) { case internal.ParsedSingleRef: - imageEncoding, err := parseImageEncoding(t.Format()) + messageEncoding, err := parseMessageEncoding(t.Format()) if err != nil { return nil, err } - return newImageRef(t, imageEncoding), nil + return newMessageRef(t, messageEncoding) case internal.ParsedArchiveRef: return newSourceRef(t), nil case internal.ParsedDirRef: @@ -258,11 +278,11 @@ func (a *refParser) GetSourceOrModuleRef( } } -func (a *refParser) GetImageRef( +func (a *refParser) GetMessageRef( ctx context.Context, value string, -) (_ ImageRef, retErr error) { - ctx, span := a.tracer.Start(ctx, "get_image_ref") +) (_ MessageRef, retErr error) { + ctx, span := a.tracer.Start(ctx, "get_message_ref") defer span.End() defer func() { if retErr != nil { @@ -270,19 +290,19 @@ func (a *refParser) GetImageRef( span.SetStatus(codes.Error, retErr.Error()) } }() - parsedRef, err := a.getParsedRef(ctx, value, imageFormats) + parsedRef, err := a.getParsedRef(ctx, value, messageFormats) if err != nil { return nil, err } parsedSingleRef, ok := parsedRef.(internal.ParsedSingleRef) if !ok { - return nil, fmt.Errorf("invalid ParsedRef type for image: %T", parsedRef) + return nil, fmt.Errorf("invalid ParsedRef type for message: %T", parsedRef) } - imageEncoding, err := parseImageEncoding(parsedSingleRef.Format()) + messageEncoding, err := parseMessageEncoding(parsedSingleRef.Format()) if err != nil { return nil, err } - return newImageRef(parsedSingleRef, imageEncoding), nil + return newMessageRef(parsedSingleRef, messageEncoding) } func (a *refParser) GetSourceRef( @@ -361,15 +381,29 @@ func (a *refParser) checkDeprecated(parsedRef internal.ParsedRef) { } } -func newRawRefProcessor() func(*internal.RawRef) error { - return func(rawRef *internal.RawRef) error { - // if format option is not set and path is "-", default to bin - var format string - var compressionType internal.CompressionType - if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { +func processRawRef(rawRef *internal.RawRef) error { + // if format option is not set and path is "-", default to bin + var format string + var compressionType internal.CompressionType + if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { + format = formatBinpb + } else { + switch filepath.Ext(rawRef.Path) { + case ".bin", ".binpb": format = formatBinpb - } else { - switch filepath.Ext(rawRef.Path) { + case ".json": + format = formatJSON + case ".tar": + format = formatTar + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + case ".zip": + format = formatZip + case ".gz": + compressionType = internal.CompressionTypeGzip + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { case ".bin", ".binpb": format = formatBinpb case ".json": @@ -380,68 +414,52 @@ func newRawRefProcessor() func(*internal.RawRef) error { format = formatTxtpb case ".yaml": format = formatYAML - case ".zip": - format = formatZip - case ".gz": - compressionType = internal.CompressionTypeGzip - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".tar": - format = formatTar - case ".txtpb": - format = formatTxtpb - case ".yaml": - format = formatYAML - default: - return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) - } - case ".zst": - compressionType = internal.CompressionTypeZstd - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".tar": - format = formatTar - case ".txtpb": - format = formatTxtpb - case ".yaml": - format = formatYAML - default: - return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) - } - case ".tgz": + default: + return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) + } + case ".zst": + compressionType = internal.CompressionTypeZstd + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".tar": format = formatTar - compressionType = internal.CompressionTypeGzip - case ".git": - format = formatGit - // This only applies if the option accept `ProtoFileRef` is passed in, otherwise - // it falls through to the `default` case. - case ".proto": - fileInfo, err := os.Stat(rawRef.Path) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("path provided is not a valid proto file: %s, %w", rawRef.Path, err) - } - if fileInfo != nil && fileInfo.IsDir() { - return fmt.Errorf("path provided is not a valid proto file: a directory named %s already exists", rawRef.Path) - } - format = formatProtoFile + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML default: - var err error - format, err = assumeModuleOrDir(rawRef.Path) - if err != nil { - return err - } + return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + } + case ".tgz": + format = formatTar + compressionType = internal.CompressionTypeGzip + case ".git": + format = formatGit + // This only applies if the option accept `ProtoFileRef` is passed in, otherwise + // it falls through to the `default` case. + case ".proto": + fileInfo, err := os.Stat(rawRef.Path) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("path provided is not a valid proto file: %s, %w", rawRef.Path, err) + } + if fileInfo != nil && fileInfo.IsDir() { + return fmt.Errorf("path provided is not a valid proto file: a directory named %s already exists", rawRef.Path) + } + format = formatProtoFile + default: + var err error + format, err = assumeModuleOrDir(rawRef.Path) + if err != nil { + return err } } - rawRef.Format = format - rawRef.CompressionType = compressionType - return nil } + rawRef.Format = format + rawRef.CompressionType = compressionType + return nil } func processRawRefSource(rawRef *internal.RawRef) error { @@ -524,39 +542,20 @@ func processRawRefSourceOrModule(rawRef *internal.RawRef) error { return nil } -func processRawRefImage(rawRef *internal.RawRef) error { - // if format option is not set and path is "-", default to bin - var format string - var compressionType internal.CompressionType - if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { - format = formatBinpb - } else { - switch filepath.Ext(rawRef.Path) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".txtpb": - format = formatTxtpb - case ".yaml": - format = formatYAML - case ".gz": - compressionType = internal.CompressionTypeGzip - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { - case ".bin", ".binpb": - format = formatBinpb - case ".json": - format = formatJSON - case ".txtpb": - format = formatTxtpb - case ".yaml": - format = formatYAML - default: - return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) - } - case ".zst": - compressionType = internal.CompressionTypeZstd - switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { +func newProcessRawRefMessage(defaultMessageEncoding MessageEncoding) func(*internal.RawRef) error { + return func(rawRef *internal.RawRef) error { + defaultFormat, ok := messageEncodingToFormat[defaultMessageEncoding] + if !ok { + // This is a system error. + return fmt.Errorf("unknown MessageEncoding: %v", defaultMessageEncoding) + } + // if format option is not set and path is "-", default to bin + var format string + var compressionType internal.CompressionType + if rawRef.Path == "-" || app.IsDevNull(rawRef.Path) || app.IsDevStdin(rawRef.Path) || app.IsDevStdout(rawRef.Path) { + format = defaultFormat + } else { + switch filepath.Ext(rawRef.Path) { case ".bin", ".binpb": format = formatBinpb case ".json": @@ -565,16 +564,42 @@ func processRawRefImage(rawRef *internal.RawRef) error { format = formatTxtpb case ".yaml": format = formatYAML + case ".gz": + compressionType = internal.CompressionTypeGzip + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + default: + return fmt.Errorf("path %q had .gz extension with unknown format", rawRef.Path) + } + case ".zst": + compressionType = internal.CompressionTypeZstd + switch filepath.Ext(strings.TrimSuffix(rawRef.Path, filepath.Ext(rawRef.Path))) { + case ".bin", ".binpb": + format = formatBinpb + case ".json": + format = formatJSON + case ".txtpb": + format = formatTxtpb + case ".yaml": + format = formatYAML + default: + return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + } default: - return fmt.Errorf("path %q had .zst extension with unknown format", rawRef.Path) + format = defaultFormat } - default: - format = formatBinpb } + rawRef.Format = format + rawRef.CompressionType = compressionType + return nil } - rawRef.Format = format - rawRef.CompressionType = compressionType - return nil } func processRawRefModule(rawRef *internal.RawRef) error { @@ -582,18 +607,18 @@ func processRawRefModule(rawRef *internal.RawRef) error { return nil } -func parseImageEncoding(format string) (ImageEncoding, error) { +func parseMessageEncoding(format string) (MessageEncoding, error) { switch format { case formatBin, formatBinpb, formatBingz: - return ImageEncodingBin, nil + return MessageEncodingBinpb, nil case formatJSON, formatJSONGZ: - return ImageEncodingJSON, nil + return MessageEncodingJSON, nil case formatTxtpb: - return ImageEncodingTxtpb, nil + return MessageEncodingTxtpb, nil case formatYAML: - return ImageEncodingYAML, nil + return MessageEncodingYAML, nil default: - return 0, fmt.Errorf("invalid format for image: %q", format) + return 0, fmt.Errorf("invalid format for message: %q", format) } } @@ -617,3 +642,13 @@ func assumeModuleOrDir(path string) (string, error) { // cannot be parsed into a module, assume dir for here return formatDir, nil } + +type messageRefParserOptions struct { + defaultMessageEncoding MessageEncoding +} + +func newMessageRefParserOptions() *messageRefParserOptions { + return &messageRefParserOptions{ + defaultMessageEncoding: MessageEncodingBinpb, + } +} diff --git a/private/buf/buffetch/ref_parser_test.go b/private/buf/buffetch/ref_parser_test.go index 64f53c9b71..1e116240a3 100644 --- a/private/buf/buffetch/ref_parser_test.go +++ b/private/buf/buffetch/ref_parser_test.go @@ -475,6 +475,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.bin", ) @@ -485,6 +486,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.bin.gz", ) @@ -495,6 +497,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.binpb", ) @@ -505,6 +508,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.binpb.gz", ) @@ -515,6 +519,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.json", ) @@ -525,6 +530,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.json.gz", ) @@ -535,6 +541,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.json.gz#compression=none", ) @@ -545,6 +552,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.json.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.json.gz#compression=gzip", ) @@ -555,6 +563,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.txtpb", ) @@ -565,6 +574,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.txtpb.gz", ) @@ -575,6 +585,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.txtpb.gz#compression=none", ) @@ -585,6 +596,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.txtpb.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.txtpb.gz#compression=gzip", ) @@ -595,6 +607,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.yaml", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.yaml", ) @@ -605,6 +618,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.yaml.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.yaml.gz", ) @@ -615,6 +629,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.yaml.gz", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/file.yaml.gz#compression=none", ) @@ -625,9 +640,46 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.yaml.gz", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/file.yaml.gz#compression=gzip", ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatYAML, + "path/to/file.yaml", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + map[string]string{ + "use_proto_names": "true", + }, + ), + "path/to/file.yaml#use_proto_names=true", + ) + testGetParsedRefError( + t, + internal.NewOptionsInvalidKeysError("use_something_else"), + "path/to/file.yaml#use_something_else=true", + ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedSingleRef( + formatJSON, + "path/to/file.json", + internal.FileSchemeLocal, + internal.CompressionTypeNone, + map[string]string{ + "use_proto_names": "true", + }, + ), + "path/to/file.json#use_proto_names=true", + ) + testGetParsedRefError( + t, + internal.NewOptionsInvalidKeysError("use_something_else"), + "path/to/file.json#use_something_else=true", + ) testGetParsedRefSuccess( t, internal.NewDirectParsedSingleRef( @@ -635,6 +687,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-", ) @@ -645,6 +698,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-#format=json", ) @@ -655,6 +709,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-#format=txtpb", ) @@ -665,6 +720,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeStdio, internal.CompressionTypeNone, + nil, ), "-#format=yaml", ) @@ -675,6 +731,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "", internal.FileSchemeNull, internal.CompressionTypeNone, + nil, ), app.DevNullFilePath, ) @@ -685,6 +742,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=bin", ) @@ -695,6 +753,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=bin,compression=none", ) @@ -705,6 +764,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/dir#format=bin,compression=gzip", ) @@ -715,6 +775,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=binpb", ) @@ -725,6 +786,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeNone, + nil, ), "path/to/dir#format=binpb,compression=none", ) @@ -735,6 +797,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/dir", internal.FileSchemeLocal, internal.CompressionTypeGzip, + nil, ), "path/to/dir#format=binpb,compression=gzip", ) @@ -979,6 +1042,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file#format=bin,compression=zstd", ) @@ -989,6 +1053,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.bin.zst", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file.bin.zst", ) @@ -999,6 +1064,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file#format=binpb,compression=zstd", ) @@ -1009,6 +1075,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "path/to/file.binpb.zst", internal.FileSchemeLocal, internal.CompressionTypeZstd, + nil, ), "path/to/file.binpb.zst", ) @@ -1047,6 +1114,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.bin", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.bin", ) @@ -1057,6 +1125,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.ext", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.ext#format=bin", ) @@ -1067,6 +1136,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.bin?private_token=bar", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.bin?private_token=bar#format=bin", ) @@ -1077,6 +1147,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.binpb", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.binpb", ) @@ -1087,6 +1158,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "github.com/path/to/file.ext", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://github.com/path/to/file.ext#format=binpb", ) @@ -1097,6 +1169,7 @@ func TestGetParsedRefSuccess(t *testing.T) { "gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.binpb?private_token=bar", internal.FileSchemeHTTPS, internal.CompressionTypeNone, + nil, ), "https://gitlab.com/api/v4/projects/foo/packages/generic/proto/0.0.1/proto.binpb?private_token=bar#format=binpb", ) @@ -1171,7 +1244,7 @@ func TestGetParsedRefError(t *testing.T) { ) testGetParsedRefError( t, - internal.NewOptionsInvalidKeyError("foo"), + internal.NewOptionsInvalidKeysError("foo"), "path/to/foo.tar.gz#foo=bar", ) testGetParsedRefError( diff --git a/private/buf/buffetch/ref_parser_unix_test.go b/private/buf/buffetch/ref_parser_unix_test.go index 6b72856608..6127b6a28b 100644 --- a/private/buf/buffetch/ref_parser_unix_test.go +++ b/private/buf/buffetch/ref_parser_unix_test.go @@ -33,6 +33,7 @@ func TestGetParsedRefSuccess_UnixOnly(t *testing.T) { "", internal.FileSchemeStdin, internal.CompressionTypeNone, + nil, ), app.DevStdinFilePath, ) @@ -43,6 +44,7 @@ func TestGetParsedRefSuccess_UnixOnly(t *testing.T) { "", internal.FileSchemeStdout, internal.CompressionTypeNone, + nil, ), app.DevStdoutFilePath, ) diff --git a/private/buf/buffetch/writer.go b/private/buf/buffetch/writer.go index 72d608ac89..f714edba54 100644 --- a/private/buf/buffetch/writer.go +++ b/private/buf/buffetch/writer.go @@ -39,10 +39,10 @@ func newWriter( } } -func (w *writer) PutImageFile( +func (w *writer) PutMessageFile( ctx context.Context, container app.EnvStdoutContainer, - imageRef ImageRef, + messageRef MessageRef, ) (io.WriteCloser, error) { - return w.internalWriter.PutFile(ctx, container, imageRef.internalFileRef()) + return w.internalWriter.PutFile(ctx, container, messageRef.internalSingleRef()) } diff --git a/private/buf/bufref/bufref.go b/private/buf/bufref/bufref.go deleted file mode 100644 index ee4e7aae53..0000000000 --- a/private/buf/bufref/bufref.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufref - -import ( - "strings" -) - -// GetRawPathAndOptions returns the raw path and options from the value provided, -// the rawPath will be non-empty when returning without error here. -func GetRawPathAndOptions(value string) (string, map[string]string, error) { - value = strings.TrimSpace(value) - if value == "" { - return "", nil, newValueEmptyError() - } - - switch splitValue := strings.Split(value, "#"); len(splitValue) { - case 1: - return value, nil, nil - case 2: - path := strings.TrimSpace(splitValue[0]) - optionsString := strings.TrimSpace(splitValue[1]) - if path == "" { - return "", nil, newValueStartsWithHashtagError(value) - } - if optionsString == "" { - return "", nil, newValueEndsWithHashtagError(value) - } - options := make(map[string]string) - for _, pair := range strings.Split(optionsString, ",") { - split := strings.Split(pair, "=") - if len(split) != 2 { - return "", nil, newOptionsInvalidError(optionsString) - } - key := strings.TrimSpace(split[0]) - value := strings.TrimSpace(split[1]) - if key == "" || value == "" { - return "", nil, newOptionsInvalidError(optionsString) - } - if _, ok := options[key]; ok { - return "", nil, newOptionsDuplicateKeyError(key) - } - options[key] = value - } - return path, options, nil - default: - return "", nil, newValueMultipleHashtagsError(value) - } -} diff --git a/private/buf/bufref/errors.go b/private/buf/bufref/errors.go deleted file mode 100644 index 6eabfc9fc2..0000000000 --- a/private/buf/bufref/errors.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufref - -import ( - "errors" - "fmt" -) - -func newValueEmptyError() error { - return errors.New("required") -} - -func newValueMultipleHashtagsError(value string) error { - return fmt.Errorf("%q has multiple #s which is invalid", value) -} - -func newValueStartsWithHashtagError(value string) error { - return fmt.Errorf("%q starts with # which is invalid", value) -} - -func newValueEndsWithHashtagError(value string) error { - return fmt.Errorf("%q ends with # which is invalid", value) -} - -func newOptionsInvalidError(s string) error { - return fmt.Errorf("invalid options: %q", s) -} - -func newOptionsDuplicateKeyError(key string) error { - return fmt.Errorf("duplicate options key: %q", key) -} diff --git a/private/buf/bufref/usage.gen.go b/private/buf/bufref/usage.gen.go deleted file mode 100644 index d5fbc2021b..0000000000 --- a/private/buf/bufref/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufref - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/bufwire/bufwire.go b/private/buf/bufwire/bufwire.go index 3394d99c2a..0109377c51 100644 --- a/private/buf/bufwire/bufwire.go +++ b/private/buf/bufwire/bufwire.go @@ -20,7 +20,6 @@ package bufwire import ( "context" - "github.com/bufbuild/buf/private/buf/bufconvert" "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufconfig" @@ -161,7 +160,7 @@ type ImageReader interface { GetImage( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, externalDirOrFilePathsAllowNotExist bool, @@ -172,7 +171,7 @@ type ImageReader interface { // NewImageReader returns a new ImageReader. func NewImageReader( logger *zap.Logger, - fetchReader buffetch.ImageReader, + fetchReader buffetch.MessageReader, ) ImageReader { return newImageReader( logger, @@ -189,7 +188,7 @@ type ImageWriter interface { PutImage( ctx context.Context, container app.EnvStdoutContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, image bufimage.Image, asFileDescriptorSet bool, excludeImports bool, @@ -215,16 +214,18 @@ type ProtoEncodingReader interface { container app.EnvStdinContainer, image bufimage.Image, typeName string, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (proto.Message, error) } // NewProtoEncodingReader returns a new ProtoEncodingReader. func NewProtoEncodingReader( logger *zap.Logger, + fetchReader buffetch.MessageReader, ) ProtoEncodingReader { return newProtoEncodingReader( logger, + fetchReader, ) } @@ -239,15 +240,17 @@ type ProtoEncodingWriter interface { container app.EnvStdoutContainer, image bufimage.Image, message proto.Message, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) error } // NewProtoEncodingWriter returns a new ProtoEncodingWriter. func NewProtoEncodingWriter( logger *zap.Logger, + fetchWriter buffetch.Writer, ) ProtoEncodingWriter { return newProtoEncodingWriter( logger, + fetchWriter, ) } diff --git a/private/buf/bufwire/file_lister.go b/private/buf/bufwire/file_lister.go index 7e8cee376b..82c98d035e 100644 --- a/private/buf/bufwire/file_lister.go +++ b/private/buf/bufwire/file_lister.go @@ -171,7 +171,7 @@ func (e *fileLister) listFilesWithoutImports( } } return fileInfos, nil, nil - case buffetch.ImageRef: + case buffetch.MessageRef: // if we have an image, list the files in the image image, err := e.imageReader.GetImage( ctx, diff --git a/private/buf/bufwire/image_config_reader.go b/private/buf/bufwire/image_config_reader.go index 5b7fee41c8..00e5c86055 100644 --- a/private/buf/bufwire/image_config_reader.go +++ b/private/buf/bufwire/image_config_reader.go @@ -78,7 +78,7 @@ func (i *imageConfigReader) GetImageConfigs( excludeSourceCodeInfo bool, ) ([]ImageConfig, []bufanalysis.FileAnnotation, error) { switch t := ref.(type) { - case buffetch.ImageRef: + case buffetch.MessageRef: env, err := i.getImageImageConfig( ctx, container, @@ -193,7 +193,7 @@ func (i *imageConfigReader) getSourceOrModuleImageConfigs( func (i *imageConfigReader) getImageImageConfig( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, configOverride string, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, @@ -203,7 +203,7 @@ func (i *imageConfigReader) getImageImageConfig( image, err := i.imageReader.GetImage( ctx, container, - imageRef, + messageRef, externalDirOrFilePaths, externalExcludeDirOrFilePaths, externalDirOrFilePathsAllowNotExist, diff --git a/private/buf/bufwire/image_reader.go b/private/buf/bufwire/image_reader.go index cd7d7d96d0..2ddc318e88 100644 --- a/private/buf/bufwire/image_reader.go +++ b/private/buf/bufwire/image_reader.go @@ -38,13 +38,13 @@ const ( type imageReader struct { logger *zap.Logger - fetchReader buffetch.ImageReader + fetchReader buffetch.MessageReader tracer trace.Tracer } func newImageReader( logger *zap.Logger, - fetchReader buffetch.ImageReader, + fetchReader buffetch.MessageReader, ) *imageReader { return &imageReader{ logger: logger.Named(loggerName), @@ -56,7 +56,7 @@ func newImageReader( func (i *imageReader) GetImage( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, externalDirOrFilePaths []string, externalExcludeDirOrFilePaths []string, externalDirOrFilePathsAllowNotExist bool, @@ -70,7 +70,7 @@ func (i *imageReader) GetImage( span.SetStatus(codes.Error, retErr.Error()) } }() - readCloser, err := i.fetchReader.GetImageFile(ctx, container, imageRef) + readCloser, err := i.fetchReader.GetMessageFile(ctx, container, messageRef) if err != nil { return nil, err } @@ -83,11 +83,11 @@ func (i *imageReader) GetImage( } protoImage := &imagev1.Image{} var imageFromProtoOptions []bufimage.NewImageForProtoOption - switch imageEncoding := imageRef.ImageEncoding(); imageEncoding { + switch messageEncoding := messageRef.MessageEncoding(); messageEncoding { // we have to double parse due to custom options // See https://github.com/golang/protobuf/issues/1123 // TODO: revisit - case buffetch.ImageEncodingBin: + case buffetch.MessageEncodingBinpb: _, span := i.tracer.Start(ctx, "wire_unmarshal") if err := protoencoding.NewWireUnmarshaler(nil).Unmarshal(data, protoImage); err != nil { span.RecordError(err) @@ -96,7 +96,7 @@ func (i *imageReader) GetImage( return nil, fmt.Errorf("could not unmarshal image: %v", err) } span.End() - case buffetch.ImageEncodingJSON: + case buffetch.MessageEncodingJSON: resolver, err := i.bootstrapResolver(ctx, protoencoding.NewJSONUnmarshaler(nil), data) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (i *imageReader) GetImage( jsonUnmarshalSpan.End() // we've already re-parsed, by unmarshalling 2x above imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) - case buffetch.ImageEncodingTxtpb: + case buffetch.MessageEncodingTxtpb: resolver, err := i.bootstrapResolver(ctx, protoencoding.NewTxtpbUnmarshaler(nil), data) if err != nil { return nil, err @@ -126,7 +126,7 @@ func (i *imageReader) GetImage( txtpbUnmarshalSpan.End() // we've already re-parsed, by unmarshalling 2x above imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) - case buffetch.ImageEncodingYAML: + case buffetch.MessageEncodingYAML: resolver, err := i.bootstrapResolver(ctx, protoencoding.NewYAMLUnmarshaler(nil), data) if err != nil { return nil, err @@ -142,7 +142,7 @@ func (i *imageReader) GetImage( // we've already re-parsed, by unmarshalling 2x above imageFromProtoOptions = append(imageFromProtoOptions, bufimage.WithNoReparse()) default: - return nil, fmt.Errorf("unknown image encoding: %v", imageEncoding) + return nil, fmt.Errorf("unknown message encoding: %v", messageEncoding) } if excludeSourceCodeInfo { for _, fileDescriptorProto := range protoImage.File { @@ -158,7 +158,7 @@ func (i *imageReader) GetImage( } imagePaths := make([]string, len(externalDirOrFilePaths)) for i, externalDirOrFilePath := range externalDirOrFilePaths { - imagePath, err := imageRef.PathForExternalPath(externalDirOrFilePath) + imagePath, err := messageRef.PathForExternalPath(externalDirOrFilePath) if err != nil { return nil, err } @@ -166,7 +166,7 @@ func (i *imageReader) GetImage( } excludePaths := make([]string, len(externalExcludeDirOrFilePaths)) for i, excludeDirOrFilePath := range externalExcludeDirOrFilePaths { - excludePath, err := imageRef.PathForExternalPath(excludeDirOrFilePath) + excludePath, err := messageRef.PathForExternalPath(excludeDirOrFilePath) if err != nil { return nil, err } diff --git a/private/buf/bufwire/image_writer.go b/private/buf/bufwire/image_writer.go index 7e9314b9e5..55fd3751d8 100644 --- a/private/buf/bufwire/image_writer.go +++ b/private/buf/bufwire/image_writer.go @@ -47,7 +47,7 @@ func newImageWriter( func (i *imageWriter) PutImage( ctx context.Context, container app.EnvStdoutContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, image bufimage.Image, asFileDescriptorSet bool, excludeImports bool, @@ -61,7 +61,7 @@ func (i *imageWriter) PutImage( } }() // stop short for performance - if imageRef.IsNull() { + if messageRef.IsNull() { return nil } writeImage := image @@ -74,11 +74,11 @@ func (i *imageWriter) PutImage( } else { message = bufimage.ImageToProtoImage(writeImage) } - data, err := i.imageMarshal(ctx, message, image, imageRef.ImageEncoding()) + data, err := i.imageMarshal(ctx, message, image, messageRef) if err != nil { return err } - writeCloser, err := i.fetchWriter.PutImageFile(ctx, container, imageRef) + writeCloser, err := i.fetchWriter.PutMessageFile(ctx, container, messageRef) if err != nil { return err } @@ -93,7 +93,7 @@ func (i *imageWriter) imageMarshal( ctx context.Context, message proto.Message, image bufimage.Image, - imageEncoding buffetch.ImageEncoding, + messageRef buffetch.MessageRef, ) (_ []byte, retErr error) { _, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "image_marshal") defer span.End() @@ -103,10 +103,10 @@ func (i *imageWriter) imageMarshal( span.SetStatus(codes.Error, retErr.Error()) } }() - switch imageEncoding { - case buffetch.ImageEncodingBin: + switch messageEncoding := messageRef.MessageEncoding(); messageEncoding { + case buffetch.MessageEncodingBinpb: return protoencoding.NewWireMarshaler().Marshal(message) - case buffetch.ImageEncodingJSON: + case buffetch.MessageEncodingJSON: // TODO: verify that image is complete resolver, err := protoencoding.NewResolver( bufimage.ImageToFileDescriptorProtos(image)..., @@ -114,8 +114,8 @@ func (i *imageWriter) imageMarshal( if err != nil { return nil, err } - return protoencoding.NewJSONMarshaler(resolver).Marshal(message) - case buffetch.ImageEncodingTxtpb: + return newJSONMarshaler(resolver, messageRef).Marshal(message) + case buffetch.MessageEncodingTxtpb: // TODO: verify that image is complete resolver, err := protoencoding.NewResolver( bufimage.ImageToFileDescriptorProtos(image)..., @@ -124,7 +124,7 @@ func (i *imageWriter) imageMarshal( return nil, err } return protoencoding.NewTxtpbMarshaler(resolver).Marshal(message) - case buffetch.ImageEncodingYAML: + case buffetch.MessageEncodingYAML: resolver, err := protoencoding.NewResolver( bufimage.ImageToFileDescriptorProtos( image, @@ -133,10 +133,8 @@ func (i *imageWriter) imageMarshal( if err != nil { return nil, err } - return protoencoding.NewYAMLMarshaler( - resolver, - protoencoding.YAMLMarshalerWithIndent(2)).Marshal(message) + return newYAMLMarshaler(resolver, messageRef).Marshal(message) default: - return nil, fmt.Errorf("unknown image encoding: %v", imageEncoding) + return nil, fmt.Errorf("unknown message encoding: %v", messageEncoding) } } diff --git a/private/buf/bufwire/module_config_reader_test.go b/private/buf/bufwire/module_config_reader_test.go index 65d7f2a7a8..d86c8ffd4a 100644 --- a/private/buf/bufwire/module_config_reader_test.go +++ b/private/buf/bufwire/module_config_reader_test.go @@ -200,10 +200,10 @@ func (r *fakeModuleFetcher) GetModule( ) } -func (r *fakeModuleFetcher) GetImageFile( +func (r *fakeModuleFetcher) GetMessageFile( ctx context.Context, container app.EnvStdinContainer, - imageRef buffetch.ImageRef, + messageRef buffetch.MessageRef, ) (io.ReadCloser, error) { return nil, nil } diff --git a/private/buf/bufwire/proto_encoding_reader.go b/private/buf/bufwire/proto_encoding_reader.go index d379dfa97f..c7e1516e50 100644 --- a/private/buf/bufwire/proto_encoding_reader.go +++ b/private/buf/bufwire/proto_encoding_reader.go @@ -19,9 +19,8 @@ import ( "errors" "fmt" "io" - "os" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufreflect" "github.com/bufbuild/buf/private/pkg/app" @@ -34,16 +33,19 @@ import ( ) type protoEncodingReader struct { - logger *zap.Logger + logger *zap.Logger + fetchReader buffetch.MessageReader } var _ ProtoEncodingReader = &protoEncodingReader{} func newProtoEncodingReader( logger *zap.Logger, + fetchReader buffetch.MessageReader, ) *protoEncodingReader { return &protoEncodingReader{ - logger: logger, + logger: logger, + fetchReader: fetchReader, } } @@ -52,7 +54,7 @@ func (p *protoEncodingReader) GetMessage( container app.EnvStdinContainer, image bufimage.Image, typeName string, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (_ proto.Message, retErr error) { ctx, span := otel.GetTracerProvider().Tracer("bufbuild/buf").Start(ctx, "get_message") defer span.End() @@ -71,13 +73,13 @@ func (p *protoEncodingReader) GetMessage( } var unmarshaler protoencoding.Unmarshaler switch messageRef.MessageEncoding() { - case bufconvert.MessageEncodingBinpb: + case buffetch.MessageEncodingBinpb: unmarshaler = protoencoding.NewWireUnmarshaler(resolver) - case bufconvert.MessageEncodingJSON: + case buffetch.MessageEncodingJSON: unmarshaler = protoencoding.NewJSONUnmarshaler(resolver) - case bufconvert.MessageEncodingTxtpb: + case buffetch.MessageEncodingTxtpb: unmarshaler = protoencoding.NewTxtpbUnmarshaler(resolver) - case bufconvert.MessageEncodingYAML: + case buffetch.MessageEncodingYAML: unmarshaler = protoencoding.NewYAMLUnmarshaler( resolver, protoencoding.YAMLUnmarshalerWithPath(messageRef.Path()), @@ -85,13 +87,9 @@ func (p *protoEncodingReader) GetMessage( default: return nil, errors.New("unknown message encoding type") } - readCloser := io.NopCloser(container.Stdin()) - if messageRef.Path() != "-" { - var err error - readCloser, err = os.Open(messageRef.Path()) - if err != nil { - return nil, err - } + readCloser, err := p.fetchReader.GetMessageFile(ctx, container, messageRef) + if err != nil { + return nil, err } defer func() { retErr = multierr.Append(retErr, readCloser.Close()) diff --git a/private/buf/bufwire/proto_encoding_writer.go b/private/buf/bufwire/proto_encoding_writer.go index 6b83a8da02..5613f29a64 100644 --- a/private/buf/bufwire/proto_encoding_writer.go +++ b/private/buf/bufwire/proto_encoding_writer.go @@ -17,12 +17,10 @@ package bufwire import ( "context" "errors" - "os" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/ioextended" "github.com/bufbuild/buf/private/pkg/protoencoding" "go.uber.org/multierr" "go.uber.org/zap" @@ -30,16 +28,19 @@ import ( ) type protoEncodingWriter struct { - logger *zap.Logger + logger *zap.Logger + fetchWriter buffetch.Writer } var _ ProtoEncodingWriter = &protoEncodingWriter{} func newProtoEncodingWriter( logger *zap.Logger, + fetchWriter buffetch.Writer, ) *protoEncodingWriter { return &protoEncodingWriter{ - logger: logger, + logger: logger, + fetchWriter: fetchWriter, } } @@ -48,7 +49,7 @@ func (p *protoEncodingWriter) PutMessage( container app.EnvStdoutContainer, image bufimage.Image, message proto.Message, - messageRef bufconvert.MessageEncodingRef, + messageRef buffetch.MessageRef, ) (retErr error) { // Currently, this support binpb and JSON format. resolver, err := protoencoding.NewResolver( @@ -59,14 +60,14 @@ func (p *protoEncodingWriter) PutMessage( } var marshaler protoencoding.Marshaler switch messageRef.MessageEncoding() { - case bufconvert.MessageEncodingBinpb: + case buffetch.MessageEncodingBinpb: marshaler = protoencoding.NewWireMarshaler() - case bufconvert.MessageEncodingJSON: - marshaler = protoencoding.NewJSONMarshaler(resolver) - case bufconvert.MessageEncodingTxtpb: + case buffetch.MessageEncodingJSON: + marshaler = newJSONMarshaler(resolver, messageRef) + case buffetch.MessageEncodingTxtpb: marshaler = protoencoding.NewTxtpbMarshaler(resolver) - case bufconvert.MessageEncodingYAML: - marshaler = protoencoding.NewYAMLMarshaler(resolver) + case buffetch.MessageEncodingYAML: + marshaler = newYAMLMarshaler(resolver, messageRef) default: return errors.New("unknown message encoding type") } @@ -74,12 +75,9 @@ func (p *protoEncodingWriter) PutMessage( if err != nil { return err } - writeCloser := ioextended.NopWriteCloser(container.Stdout()) - if messageRef.Path() != "-" { - writeCloser, err = os.Create(messageRef.Path()) - if err != nil { - return err - } + writeCloser, err := p.fetchWriter.PutMessageFile(ctx, container, messageRef) + if err != nil { + return err } defer func() { retErr = multierr.Append(retErr, writeCloser.Close()) diff --git a/private/buf/bufwire/util.go b/private/buf/bufwire/util.go new file mode 100644 index 0000000000..34babfea9b --- /dev/null +++ b/private/buf/bufwire/util.go @@ -0,0 +1,64 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufwire + +import ( + "github.com/bufbuild/buf/private/buf/buffetch" + "github.com/bufbuild/buf/private/pkg/protoencoding" +) + +func newJSONMarshaler( + resolver protoencoding.Resolver, + messageRef buffetch.MessageRef, +) protoencoding.Marshaler { + jsonMarshalerOptions := []protoencoding.JSONMarshalerOption{ + //protoencoding.JSONMarshalerWithIndent(), + } + if messageRef.UseProtoNames() { + jsonMarshalerOptions = append( + jsonMarshalerOptions, + protoencoding.JSONMarshalerWithUseProtoNames(), + ) + } + if messageRef.UseEnumNumbers() { + jsonMarshalerOptions = append( + jsonMarshalerOptions, + protoencoding.JSONMarshalerWithUseEnumNumbers(), + ) + } + return protoencoding.NewJSONMarshaler(resolver, jsonMarshalerOptions...) +} + +func newYAMLMarshaler( + resolver protoencoding.Resolver, + messageRef buffetch.MessageRef, +) protoencoding.Marshaler { + yamlMarshalerOptions := []protoencoding.YAMLMarshalerOption{ + protoencoding.YAMLMarshalerWithIndent(), + } + if messageRef.UseProtoNames() { + yamlMarshalerOptions = append( + yamlMarshalerOptions, + protoencoding.YAMLMarshalerWithUseProtoNames(), + ) + } + if messageRef.UseEnumNumbers() { + yamlMarshalerOptions = append( + yamlMarshalerOptions, + protoencoding.YAMLMarshalerWithUseEnumNumbers(), + ) + } + return protoencoding.NewYAMLMarshaler(resolver, yamlMarshalerOptions...) +} diff --git a/private/buf/cmd/buf/command/alpha/protoc/flags.go b/private/buf/cmd/buf/command/alpha/protoc/flags.go index ece86bb5c1..37f56b3bc1 100644 --- a/private/buf/cmd/buf/command/alpha/protoc/flags.go +++ b/private/buf/cmd/buf/command/alpha/protoc/flags.go @@ -125,7 +125,7 @@ func (f *flagsBuilder) Bind(flagSet *pflag.FlagSet) { "", fmt.Sprintf( `The location to write the FileDescriptorSet. Must be one of format %s.`, - buffetch.ImageFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( diff --git a/private/buf/cmd/buf/command/alpha/protoc/protoc.go b/private/buf/cmd/buf/command/alpha/protoc/protoc.go index 8c668dbea3..3711476db0 100644 --- a/private/buf/cmd/buf/command/alpha/protoc/protoc.go +++ b/private/buf/cmd/buf/command/alpha/protoc/protoc.go @@ -239,13 +239,13 @@ func run( if env.Output == "" { return appcmd.NewInvalidArgumentErrorf("required flag %q not set", outputFlagName) } - imageRef, err := buffetch.NewImageRefParser(container.Logger()).GetImageRef(ctx, env.Output) + messageRef, err := buffetch.NewMessageRefParser(container.Logger()).GetMessageRef(ctx, env.Output) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } return bufcli.NewWireImageWriter(container.Logger()).PutImage(ctx, container, - imageRef, + messageRef, image, true, !env.IncludeImports, diff --git a/private/buf/cmd/buf/command/build/build.go b/private/buf/cmd/buf/command/build/build.go index 25274dfe9b..678964925d 100644 --- a/private/buf/cmd/buf/command/build/build.go +++ b/private/buf/cmd/buf/command/build/build.go @@ -108,7 +108,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { app.DevNullFilePath, fmt.Sprintf( `The output location for the built image. Must be one of format %s`, - buffetch.ImageFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( @@ -155,7 +155,7 @@ func run( if err != nil { return err } - imageRef, err := buffetch.NewImageRefParser(container.Logger()).GetImageRef(ctx, flags.Output) + messageRef, err := buffetch.NewMessageRefParser(container.Logger()).GetMessageRef(ctx, flags.Output) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } @@ -170,7 +170,7 @@ func run( ).PutImage( ctx, container, - imageRef, + messageRef, image, flags.AsFileDescriptorSet, flags.ExcludeImports, diff --git a/private/buf/cmd/buf/command/convert/convert.go b/private/buf/cmd/buf/command/convert/convert.go index e0c073c200..45fbfd9247 100644 --- a/private/buf/cmd/buf/command/convert/convert.go +++ b/private/buf/cmd/buf/command/convert/convert.go @@ -20,22 +20,24 @@ import ( "fmt" "github.com/bufbuild/buf/private/buf/bufcli" - "github.com/bufbuild/buf/private/buf/bufconvert" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" "github.com/bufbuild/buf/private/gen/data/datawkt" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appflag" + "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/spf13/cobra" "github.com/spf13/pflag" ) const ( - errorFormatFlagName = "error-format" - typeFlagName = "type" - fromFlagName = "from" - outputFlagName = "to" + errorFormatFlagName = "error-format" + typeFlagName = "type" + fromFlagName = "from" + outputFlagName = "to" + disableSymlinksFlagName = "disable-symlinks" ) // NewCommand returns a new Command. @@ -90,10 +92,11 @@ Use a module on the bsr: } type flags struct { - ErrorFormat string - Type string - From string - To string + ErrorFormat string + Type string + From string + To string + DisableSymlinks bool // special InputHashtag string @@ -105,6 +108,7 @@ func newFlags() *flags { func (f *flags) Bind(flagSet *pflag.FlagSet) { bufcli.BindInputHashtag(flagSet, &f.InputHashtag) + bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) flagSet.StringVar( &f.ErrorFormat, errorFormatFlagName, @@ -126,7 +130,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { "-", fmt.Sprintf( `The location of the payload to be converted. Supported formats are %s`, - bufconvert.MessageEncodingFormatsString, + buffetch.MessageFormatsString, ), ) flagSet.StringVar( @@ -135,7 +139,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { "-", fmt.Sprintf( `The output location of the conversion. Supported formats are %s`, - bufconvert.MessageEncodingFormatsString, + buffetch.MessageFormatsString, ), ) } @@ -189,12 +193,21 @@ func run( if inputErr != nil && image == nil { return inputErr } - fromMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.From, bufconvert.MessageEncodingBinpb) + fromMessageRef, err := buffetch.NewMessageRefParser( + container.Logger(), + buffetch.MessageRefParserWithDefaultMessageEncoding( + buffetch.MessageEncodingBinpb, + ), + ).GetMessageRef(ctx, flags.From) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } + storageosProvider := bufcli.NewStorageosProvider(flags.DisableSymlinks) + runner := command.NewRunner() message, err := bufcli.NewWireProtoEncodingReader( container.Logger(), + storageosProvider, + runner, ).GetMessage( ctx, container, @@ -209,7 +222,12 @@ func run( if err != nil { return err } - outputMessageRef, err := bufconvert.NewMessageEncodingRef(ctx, flags.To, defaultToEncoding) + toMessageRef, err := buffetch.NewMessageRefParser( + container.Logger(), + buffetch.MessageRefParserWithDefaultMessageEncoding( + defaultToEncoding, + ), + ).GetMessageRef(ctx, flags.To) if err != nil { return fmt.Errorf("--%s: %v", outputFlagName, err) } @@ -220,22 +238,22 @@ func run( container, image, message, - outputMessageRef, + toMessageRef, ) } // inverseEncoding returns the opposite encoding of the provided encoding, // which will be the default output encoding for a given payload encoding. -func inverseEncoding(encoding bufconvert.MessageEncoding) (bufconvert.MessageEncoding, error) { +func inverseEncoding(encoding buffetch.MessageEncoding) (buffetch.MessageEncoding, error) { switch encoding { - case bufconvert.MessageEncodingBinpb: - return bufconvert.MessageEncodingJSON, nil - case bufconvert.MessageEncodingJSON: - return bufconvert.MessageEncodingBinpb, nil - case bufconvert.MessageEncodingTxtpb: - return bufconvert.MessageEncodingBinpb, nil - case bufconvert.MessageEncodingYAML: - return bufconvert.MessageEncodingBinpb, nil + case buffetch.MessageEncodingBinpb: + return buffetch.MessageEncodingJSON, nil + case buffetch.MessageEncodingJSON: + return buffetch.MessageEncodingBinpb, nil + case buffetch.MessageEncodingTxtpb: + return buffetch.MessageEncodingBinpb, nil + case buffetch.MessageEncodingYAML: + return buffetch.MessageEncodingBinpb, nil default: return 0, fmt.Errorf("unknown message encoding %v", encoding) } diff --git a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go index e0dfeb7c1c..268f7e452f 100644 --- a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go +++ b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go @@ -94,7 +94,7 @@ func handle( if !externalConfig.LimitToInputFiles { files = nil } - againstImageRef, err := buffetch.NewImageRefParser(logger).GetImageRef(ctx, externalConfig.AgainstInput) + againstMessageRef, err := buffetch.NewMessageRefParser(logger).GetMessageRef(ctx, externalConfig.AgainstInput) if err != nil { return fmt.Errorf("against_input: %v", err) } @@ -104,7 +104,7 @@ func handle( againstImage, err := imageReader.GetImage( ctx, newContainer(container), - againstImageRef, + againstMessageRef, files, // limit to the input files if specified nil, // exclude paths are not supported on this plugin true, // allow files in the against input to not exist diff --git a/private/pkg/protoencoding/json_marshaler.go b/private/pkg/protoencoding/json_marshaler.go index 22cba9cd9e..d12c306984 100644 --- a/private/pkg/protoencoding/json_marshaler.go +++ b/private/pkg/protoencoding/json_marshaler.go @@ -26,6 +26,7 @@ type jsonMarshaler struct { resolver Resolver indent string useProtoNames bool + useEnumNumbers bool emitUnpopulated bool } @@ -46,6 +47,7 @@ func (m *jsonMarshaler) Marshal(message proto.Message) ([]byte, error) { options := protojson.MarshalOptions{ Resolver: m.resolver, UseProtoNames: m.useProtoNames, + UseEnumNumbers: m.useEnumNumbers, EmitUnpopulated: m.emitUnpopulated, } data, err := options.Marshal(message) diff --git a/private/pkg/protoencoding/protoencoding.go b/private/pkg/protoencoding/protoencoding.go index 0313d41242..f384cf8f90 100644 --- a/private/pkg/protoencoding/protoencoding.go +++ b/private/pkg/protoencoding/protoencoding.go @@ -89,13 +89,20 @@ func JSONMarshalerWithIndent() JSONMarshalerOption { } } -// JSONMarshalerWithUseProtoNames says to use an use proto names. +// JSONMarshalerWithUseProtoNames says to use proto names. func JSONMarshalerWithUseProtoNames() JSONMarshalerOption { return func(jsonMarshaler *jsonMarshaler) { jsonMarshaler.useProtoNames = true } } +// JSONMarshalerWithUseEnumNumbers says to use enum numbers. +func JSONMarshalerWithUseEnumNumbers() JSONMarshalerOption { + return func(jsonMarshaler *jsonMarshaler) { + jsonMarshaler.useEnumNumbers = true + } +} + // JSONMarshalerWithEmitUnpopulated says to emit unpopulated values func JSONMarshalerWithEmitUnpopulated() JSONMarshalerOption { return func(jsonMarshaler *jsonMarshaler) { @@ -120,20 +127,27 @@ func NewYAMLMarshaler(resolver Resolver, options ...YAMLMarshalerOption) Marshal // YAMLMarshalerOption is an option for a new YAMLMarshaler. type YAMLMarshalerOption func(*yamlMarshaler) -// YAMLMarshalerWithIndent says to use an indent of the given number spaces. -func YAMLMarshalerWithIndent(indent int) YAMLMarshalerOption { +// YAMLMarshalerWithIndent says to use an indent of two spaces. +func YAMLMarshalerWithIndent() YAMLMarshalerOption { return func(yamlMarshaler *yamlMarshaler) { - yamlMarshaler.indent = indent + yamlMarshaler.indent = 2 } } -// YAMLMarshalerWithUseProtoNames says to use an use proto names. +// YAMLMarshalerWithUseProtoNames says to use proto names. func YAMLMarshalerWithUseProtoNames() YAMLMarshalerOption { return func(yamlMarshaler *yamlMarshaler) { yamlMarshaler.useProtoNames = true } } +// YAMLMarshalerWithUseEnumNumbers says to use enum numbers. +func YAMLMarshalerWithUseEnumNumbers() YAMLMarshalerOption { + return func(yamlMarshaler *yamlMarshaler) { + yamlMarshaler.useEnumNumbers = true + } +} + // YAMLMarshalerWithEmitUnpopulated says to emit unpopulated values func YAMLMarshalerWithEmitUnpopulated() YAMLMarshalerOption { return func(yamlMarshaler *yamlMarshaler) { diff --git a/private/pkg/protoencoding/yaml_marshaler.go b/private/pkg/protoencoding/yaml_marshaler.go index 7fbe14e444..e65b4b6271 100644 --- a/private/pkg/protoencoding/yaml_marshaler.go +++ b/private/pkg/protoencoding/yaml_marshaler.go @@ -23,6 +23,7 @@ type yamlMarshaler struct { resolver Resolver indent int useProtoNames bool + useEnumNumbers bool emitUnpopulated bool } @@ -44,6 +45,7 @@ func (m *yamlMarshaler) Marshal(message proto.Message) ([]byte, error) { Indent: m.indent, Resolver: m.resolver, UseProtoNames: m.useProtoNames, + UseEnumNumbers: m.useEnumNumbers, EmitUnpopulated: m.emitUnpopulated, } data, err := options.Marshal(message)