Skip to content

Commit

Permalink
Add features files to the set of standard imports (bufbuild#295)
Browse files Browse the repository at this point in the history
The "standard imports" (made available to compile operations using
`protocompile.WithStandardImports`) are files that would be included
with `protoc`, if a user were instead compiling with `protoc`.

As of v26.1, `protoc` now includes two new files:
"goole/protobuf/cpp_features.proto" and
"google/protobuf/java_features.proto". But these are _not_ generated to
Go code into packages in the Protobuf runtime, unlike all of the other
well-known imports. So, for these, we embed binary-encoded file
descriptors.

Notably `protoc` does **not** include
"google/protobuf/go_features.proto". However, that file _is_ part of the
Go Protobuf runtime, with its generated code being available via the
`google.golang.org/protobuf/types/gofeaturespb` package. So we also make
that file available via `protocompile.WithStandardImports` (even though
`protoc` doesn't include it).

(cherry picked from commit 83dc971)
  • Loading branch information
jhump authored and kralicky committed Jun 8, 2024
1 parent 70bb054 commit 7d95ff6
Show file tree
Hide file tree
Showing 9 changed files with 182 additions and 11 deletions.
10 changes: 9 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ lintfix: $(BIN)/golangci-lint ## Automatically fix some lint errors
cd internal/benchmarks && $(BIN)/golangci-lint run --fix

.PHONY: generate
generate: $(BIN)/license-header $(BIN)/goyacc test-descriptors ## Regenerate code and licenses
generate: $(BIN)/license-header $(BIN)/goyacc test-descriptors ext-features-descriptors ## Regenerate code and licenses
PATH="$(BIN)$(PATH_SEP)$(PATH)" $(GO) generate ./...
@# We want to operate on a list of modified and new files, excluding
@# deleted and ignored files. git-ls-files can't do this alone. comm -23 takes
Expand Down Expand Up @@ -186,3 +186,11 @@ test-descriptors: internal/testdata/options/options.protoset
test-descriptors: internal/testdata/options/test.protoset
test-descriptors: internal/testdata/options/test_proto3.protoset
test-descriptors: internal/testdata/options/test_editions.protoset

internal/featuresext/cpp_features.protoset: $(PROTOC)
cd $(@D) && $(PROTOC) --experimental_editions --descriptor_set_out=$(@F) google/protobuf/cpp_features.proto
internal/featuresext/java_features.protoset: $(PROTOC)
cd $(@D) && $(PROTOC) --experimental_editions --descriptor_set_out=$(@F) google/protobuf/java_features.proto

.PHONY: ext-features-descriptors
ext-features-descriptors: internal/featuresext/cpp_features.protoset internal/featuresext/java_features.protoset
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ require (
github.com/google/go-cmp v0.6.0
github.com/kralicky/codegen v0.0.0-20240307225947-51de80fcb2f3
github.com/plar/go-adaptive-radix-tree v1.0.5
github.com/stretchr/testify v1.8.4
golang.org/x/sync v0.6.0
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
github.com/stretchr/testify v1.9.0
golang.org/x/sync v0.7.0
google.golang.org/protobuf v1.33.1-0.20240422163739-e4ad8f9dfc8b
)

require (
Expand Down
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 h1:V7Da7qt0MkY3noVANIMVBk28nOnijADeOR3i5Hcvpj4=
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
google.golang.org/protobuf v1.33.1-0.20240422163739-e4ad8f9dfc8b h1:LtLYFcFBR7ZbrgFuP9p4Za5dPr/velw2v5t9Z9HFPiQ=
google.golang.org/protobuf v1.33.1-0.20240422163739-e4ad8f9dfc8b/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
7 changes: 7 additions & 0 deletions internal/featuresext/cpp_features.protoset
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

�
"google/protobuf/cpp_features.protopb google/protobuf/descriptor.proto"_
CppFeaturesP
legacy_closed_enum (B"���� true��
false�RlegacyClosedEnum:?
cpp.google.protobuf.FeatureSet� ( 2.pb.CppFeaturesRcpp
84 changes: 84 additions & 0 deletions internal/featuresext/featuresext.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright 2020-2024 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package featuresext provides file descriptors for the
// "google/protobuf/cpp_features.proto" and "google/protobuf/java_features.proto"
// standard import files. Unlike the other standard/well-known
// imports, these files have no standard Go package in their
// runtime with generated code. So in order to make them available
// as "standard imports" to compiler users, we must embed these
// descriptors into a Go package.
package featuresext

import (
_ "embed"
"fmt"
"sync"

"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/descriptorpb"
)

var (
//go:embed cpp_features.protoset
cppFeatures []byte

//go:embed java_features.protoset
javaFeatures []byte

initOnce sync.Once
initCppFeatures protoreflect.FileDescriptor
initCppErr error
initJavaFeatures protoreflect.FileDescriptor
initJavaErr error
)

func initDescriptors() {
initOnce.Do(func() {
initCppFeatures, initCppErr = buildDescriptor("google/protobuf/cpp_features.proto", cppFeatures)
initJavaFeatures, initJavaErr = buildDescriptor("google/protobuf/java_features.proto", javaFeatures)
})
}

func CppFeaturesDescriptor() (protoreflect.FileDescriptor, error) {
initDescriptors()
return initCppFeatures, initCppErr
}

func JavaFeaturesDescriptor() (protoreflect.FileDescriptor, error) {
initDescriptors()
return initJavaFeatures, initJavaErr
}

func buildDescriptor(name string, data []byte) (protoreflect.FileDescriptor, error) {
var files descriptorpb.FileDescriptorSet
err := proto.Unmarshal(data, &files)
if err != nil {
return nil, fmt.Errorf("failed to load descriptor for %q: %w", name, err)
}
if len(files.File) != 1 {
return nil, fmt.Errorf("failed to load descriptor for %q: expected embedded descriptor set to contain exactly one file but it instead has %d", name, len(files.File))
}
if files.File[0].GetName() != name {
return nil, fmt.Errorf("failed to load descriptor for %q: embedded descriptor contains wrong file %q", name, files.File[0].GetName())
}
descriptor, err := protodesc.NewFile(files.File[0], protoregistry.GlobalFiles)
if err != nil {
return nil, fmt.Errorf("failed to load descriptor for %q: %w", name, err)
}
return descriptor, nil
}
37 changes: 37 additions & 0 deletions internal/featuresext/featuresext_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2020-2024 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package featuresext

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/reflect/protoreflect"
)

func TestFeaturesExt(t *testing.T) {
t.Parallel()

file, err := CppFeaturesDescriptor()
require.NoError(t, err)
assert.Equal(t, protoreflect.FullName("pb"), file.Package())
assert.NotNil(t, file.Extensions().ByName("cpp"))

file, err = JavaFeaturesDescriptor()
require.NoError(t, err)
assert.Equal(t, protoreflect.FullName("pb"), file.Package())
assert.NotNil(t, file.Extensions().ByName("java"))
}
Binary file added internal/featuresext/java_features.protoset
Binary file not shown.
36 changes: 35 additions & 1 deletion std_imports.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/descriptorpb"
_ "google.golang.org/protobuf/types/known/anypb" // link in packages that include the standard protos included with protoc.
_ "google.golang.org/protobuf/types/gofeaturespb" // link in packages that include the standard protos included with protoc.
_ "google.golang.org/protobuf/types/known/anypb"
_ "google.golang.org/protobuf/types/known/apipb"
_ "google.golang.org/protobuf/types/known/durationpb"
_ "google.golang.org/protobuf/types/known/emptypb"
Expand All @@ -31,6 +32,8 @@ import (
_ "google.golang.org/protobuf/types/known/typepb"
_ "google.golang.org/protobuf/types/known/wrapperspb"
_ "google.golang.org/protobuf/types/pluginpb"

"github.com/kralicky/protocompile/internal/featuresext"
)

// All files that are included with protoc are also included with this package
Expand All @@ -49,6 +52,7 @@ func init() {
"google/protobuf/duration.proto",
"google/protobuf/empty.proto",
"google/protobuf/field_mask.proto",
"google/protobuf/go_features.proto",
"google/protobuf/source_context.proto",
"google/protobuf/struct.proto",
"google/protobuf/timestamp.proto",
Expand All @@ -70,6 +74,36 @@ func init() {
wellKnownMessages[msg.FullName()] = true
}
}

otherFeatures := []struct {
Name string
GetDescriptor func() (protoreflect.FileDescriptor, error)
}{
{
Name: "google/protobuf/cpp_features.proto",
GetDescriptor: featuresext.CppFeaturesDescriptor,
},
{
Name: "google/protobuf/java_features.proto",
GetDescriptor: featuresext.JavaFeaturesDescriptor,
},
}
for _, feature := range otherFeatures {
// First see if the program has generated Go code for this
// file linked in:
fd, err := protoregistry.GlobalFiles.FindFileByPath(feature.Name)
if err == nil {
standardImports[feature.Name] = protodesc.ToFileDescriptorProto(fd)
continue
}
fd, err = feature.GetDescriptor()
if err != nil {
// For these extensions to FeatureSet, we are lenient. If
// we can't load them, just ignore them.
continue
}
standardImports[feature.Name] = protodesc.ToFileDescriptorProto(fd)
}
}

func IsWellKnownType(name protoreflect.FullName) bool {
Expand Down
1 change: 1 addition & 0 deletions std_imports_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func TestStdImports(t *testing.T) {
c := Compiler{Resolver: WithStandardImports(&SourceResolver{})}
ctx := context.Background()
for name, fileProto := range standardImports {
t.Log(name)
fds, err := c.Compile(ctx, ResolvedPath(name))
if err != nil {
t.Errorf("failed to compile %q: %v", name, err)
Expand Down

0 comments on commit 7d95ff6

Please sign in to comment.