add wellknownimports package, for providing actual source code

Makefile

@@ -134,6 +134,13 @@ $(PROTOC): $(CACHE)/protoc-$(PROTOC_VERSION).zip
 	unzip -o -q $< -d $(PROTOC_DIR) && \
 	touch $@
 
+.PHONY: wellknownimports
+wellknownimports: $(PROTOC) $(sort $(wildcard $(PROTOC_DIR)/include/google/protobuf/*.proto)) $(sort $(wildcard $(PROTOC_DIR)/include/google/protobuf/*/*.proto))
+	@rm -rf wellknownimports/google 2>/dev/null && true
+	@mkdir -p wellknownimports/google/protobuf/compiler
+	cp -R $(PROTOC_DIR)/include/google/protobuf/*.proto wellknownimports/google/protobuf
+	cp -R $(PROTOC_DIR)/include/google/protobuf/compiler/*.proto wellknownimports/google/protobuf/compiler
+
 internal/testdata/all.protoset: $(PROTOC) $(sort $(wildcard internal/testdata/*.proto))
 	cd $(@D) && $(PROTOC) --descriptor_set_out=$(@F) --include_imports -I. $(filter-out protoc,$(^F))
 

resolver.go

@@ -180,6 +180,27 @@ func SourceAccessorFromMap(srcs map[string]string) func(string) (io.ReadCloser,
 
 // WithStandardImports returns a new resolver that knows about the same standard
 // imports that are included with protoc.
+//
+// Note that this uses the descriptors embedded in generated code in the packages
+// of the Protobuf Go module, except for "google/protobuf/cpp_features.proto" and
+// "google/protobuf/java_features.proto". For those two files, compiled descriptors
+// are embedded in this module because there is no package in the Protobuf Go module
+// that contains generated code for those files. This resolver also provides results
+// for the "google/protobuf/go_features.proto", which is technically not a standard
+// file (it is not included with protoc) but is included in generated code in the
+// Protobuf Go module.
+//
+// As of v0.14.0 of this module (and v1.34.2 of the Protobuf Go module and v27.0 of
+// Protobuf), the contents of the standard import "google/protobuf/descriptor.proto"
+// contain extension declarations which are *absent* from the descriptors that this
+// resolver returns. That is because extension declarations are only retained in
+// source, not at runtime, which means they are not available in the embedded
+// descriptors in generated code.
+//
+// To use versions of the standard imports that *do* include these extension
+// declarations, see wellknownimports.WithStandardImports instead. As of this
+// writing, the declarations are only needed to prevent source files from
+// illegally re-defining the custom features for C++, Java, and Go.
 func WithStandardImports(r Resolver) Resolver {
 	return ResolverFunc(func(name string) (SearchResult, error) {
 		res, err := r.FindFileByPath(name)

wellknownimports/wellknownimports.go

@@ -0,0 +1,54 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package wellknownimports provides source code for the well-known import
+// files for use with a protocompile.Compiler.
+package wellknownimports
+
+import (
+	"embed"
+	"io"
+
+	"github.com/bufbuild/protocompile"
+)
+
+//go:embed google/protobuf/*.proto google/protobuf/*/*.proto
+var files embed.FS
+
+// WithStandardImports returns a new resolver that can provide the source code for the
+// standard imports that are included with protoc. This differs from
+// protocompile.WithStandardImports, which uses descriptors embedded in generated
+// code in the Protobuf Go module. That function is lighter weight, and does not need
+// to bring in additional embedded data outside the Protobuf Go runtime. This version
+// includes its own embedded versions of the source files.
+//
+// Unlike protocompile.WithStandardImports, this resolver does provide results for the
+// "google/protobuf/cpp_features.proto" and "google/protobuf/java_features.proto" files.
+// It does not provide results for "google/protobuf/go_features.proto", since this
+// resolver is backed by source files that are shipped with the Protobuf installation.
+//
+// It is possible that the source code provided by this resolver differs from the
+// source code used to build embedded descriptors provided by protocompile.WithStandardImports.
+// That is because that other function depends on the Protobuf Go module, which could
+// resolve in user programs to a different version than was used to build this package.
+func WithStandardImports(resolver protocompile.Resolver) protocompile.Resolver {
+	return protocompile.CompositeResolver{
+		resolver,
+		&protocompile.SourceResolver{
+			Accessor: func(path string) (io.ReadCloser, error) {
+				return files.Open(path)
+			},
+		},
+	}
+}

wellknownimports/wellknownimports_test.go

@@ -0,0 +1,113 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wellknownimports
+
+import (
+	"context"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/types/descriptorpb"
+
+	"github.com/bufbuild/protocompile"
+	"github.com/bufbuild/protocompile/linker"
+)
+
+func TestWithStandardImports(t *testing.T) {
+	t.Parallel()
+	wellKnownImports := []string{
+		"google/protobuf/any.proto",
+		"google/protobuf/api.proto",
+		"google/protobuf/compiler/plugin.proto",
+		"google/protobuf/cpp_features.proto",
+		"google/protobuf/descriptor.proto",
+		"google/protobuf/duration.proto",
+		"google/protobuf/empty.proto",
+		"google/protobuf/field_mask.proto",
+		"google/protobuf/java_features.proto",
+		"google/protobuf/source_context.proto",
+		"google/protobuf/struct.proto",
+		"google/protobuf/timestamp.proto",
+		"google/protobuf/type.proto",
+		"google/protobuf/wrappers.proto",
+	}
+	// make sure we can successfully compile them all
+	c := protocompile.Compiler{
+		Resolver: WithStandardImports(&protocompile.SourceResolver{
+			Accessor: func(path string) (io.ReadCloser, error) {
+				return nil, os.ErrNotExist
+			},
+		}),
+		RetainASTs: true,
+	}
+	ctx := context.Background()
+	for _, name := range wellKnownImports {
+		t.Log(name)
+		fds, err := c.Compile(ctx, name)
+		if err != nil {
+			t.Errorf("failed to compile %q: %v", name, err)
+			continue
+		}
+		if len(fds) != 1 {
+			t.Errorf("Compile returned wrong number of descriptors: expecting 1, got %d", len(fds))
+			continue
+		}
+		// Make sure they were built from source
+		result, ok := fds[0].(linker.Result)
+		require.True(t, ok)
+		require.NotNil(t, result.AST())
+
+		if name == "google/protobuf/descriptor.proto" {
+			// verify the extension declarations are present
+			d := fds[0].FindDescriptorByName("google.protobuf.FeatureSet")
+			require.NotNil(t, d)
+			md, ok := d.(protoreflect.MessageDescriptor)
+			require.True(t, ok)
+			var extRangeCount int
+			for i := 0; i < md.ExtensionRanges().Len(); i++ {
+				opts, ok := md.ExtensionRangeOptions(i).(*descriptorpb.ExtensionRangeOptions)
+				require.True(t, ok)
+				extRangeCount += len(opts.GetDeclaration())
+			}
+			require.Greater(t, extRangeCount, 0, "no declarations found for FeatureSet for %q", name)
+		}
+	}
+}
+
+func TestCantRedefineWellKnownCustomFeature(t *testing.T) {
+	c := protocompile.Compiler{
+		Resolver: WithStandardImports(&protocompile.SourceResolver{
+			Accessor: protocompile.SourceAccessorFromMap(map[string]string{
+				"features.proto": `
+					edition = "2023";
+					import "google/protobuf/descriptor.proto";
+					message Custom {
+						bool flag = 1;
+					}
+					extend google.protobuf.FeatureSet {
+						// tag 1000 is declared by pb.cpp so shouldn't be allowed
+						Custom custom = 1000;
+					}
+					`,
+			}),
+		}),
+	}
+	ctx := context.Background()
+	_, err := c.Compile(ctx, "features.proto")
+	require.ErrorContains(t, err, `features.proto:9:56: expected extension with number 1000 to be named pb.cpp, not custom, per declaration at google/protobuf/descriptor.proto`)
+}