upgrade grpc-gateway from v1 to v2

Signed-off-by: Wei Fu <fuweid89@gmail.com>
Signed-off-by: Benjamin Wang <wachao@vmware.com>

api/authpb/auth.proto

@@ -3,6 +3,8 @@ package authpb;
 
 import "gogoproto/gogo.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/authpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

api/etcdserverpb/etcdserver.proto

@@ -3,6 +3,8 @@ package etcdserverpb;
 
 import "gogoproto/gogo.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

api/etcdserverpb/raft_internal.proto

@@ -7,6 +7,8 @@ import "rpc.proto";
 import "etcd/api/versionpb/version.proto";
 import "etcd/api/membershippb/membership.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

api/etcdserverpb/rpc.proto

@@ -8,12 +8,14 @@ import "etcd/api/versionpb/version.proto";
 
 // for grpc-gateway
 import "google/api/annotations.proto";
-import "protoc-gen-swagger/options/annotations.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
+
+option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb";
 
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.unmarshaler_all) = true;
 
-option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
   security_definitions: {
     security: {
       key: "ApiKey";

api/membershippb/membership.proto

@@ -4,6 +4,8 @@ package membershippb;
 import "gogoproto/gogo.proto";
 import "etcd/api/versionpb/version.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/membershippb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

api/mvccpb/kv.proto

@@ -3,6 +3,8 @@ package mvccpb;
 
 import "gogoproto/gogo.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/mvccpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

api/versionpb/version.proto

@@ -4,6 +4,8 @@ package versionpb;
 import "gogoproto/gogo.proto";
 import "google/protobuf/descriptor.proto";
 
+option go_package = "go.etcd.io/etcd/api/v3/versionpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.unmarshaler_all) = true;
 

scripts/genproto.sh

@@ -20,17 +20,38 @@ if [[ $(protoc --version | cut -f2 -d' ') != "3.20.3" ]]; then
 fi
 
 GOFAST_BIN=$(tool_get_bin github.com/gogo/protobuf/protoc-gen-gofast)
-GRPC_GATEWAY_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)
-SWAGGER_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger)
+GRPC_GATEWAY_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway)
+OPENAPIV2_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2)
 GOGOPROTO_ROOT="$(tool_pkg_dir github.com/gogo/protobuf/proto)/.."
-GRPC_GATEWAY_ROOT="$(tool_pkg_dir github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)/.."
+GRPC_GATEWAY_ROOT="$(tool_pkg_dir github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway)/.."
 RAFT_ROOT="$(tool_pkg_dir go.etcd.io/raft/v3/raftpb)/.."
+GOOGLEAPI_ROOT=$(mktemp -d -t 'googleapi.XXXXX')
+
+readonly googleapi_commit=0adf469dcd7822bf5bc058a7b0217f5558a75643
+
+function cleanup_googleapi() {
+  rm -rf "${GOOGLEAPI_ROOT}"
+}
+
+trap cleanup_googleapi EXIT
+
+# TODO(ahrtr): use buf (https://github.com/bufbuild/buf) to manage the protobuf dependencies?
+function download_googleapi() {
+  run pushd "${GOOGLEAPI_ROOT}"
+  run git init
+  run git remote add upstream https://github.com/googleapis/googleapis.git
+  run git fetch upstream "${googleapi_commit}"
+  run git reset --hard FETCH_HEAD
+  run popd
+}
+
+download_googleapi
 
 echo
 echo "Resolved binary and packages versions:"
 echo "  - protoc-gen-gofast:       ${GOFAST_BIN}"
 echo "  - protoc-gen-grpc-gateway: ${GRPC_GATEWAY_BIN}"
-echo "  - swagger:                 ${SWAGGER_BIN}"
+echo "  - openapiv2:               ${OPENAPIV2_BIN}"
 echo "  - gogoproto-root:          ${GOGOPROTO_ROOT}"
 echo "  - grpc-gateway-root:       ${GRPC_GATEWAY_ROOT}"
 echo "  - raft-root:               ${RAFT_ROOT}"
@@ -43,15 +64,11 @@ log_callout -e "\\nRunning gofast (gogo) proto generation..."
 
 for dir in ${DIRS}; do
   run pushd "${dir}"
-    run protoc --gofast_out=plugins=grpc:. -I=".:${GOGOPROTO_PATH}:${ETCD_ROOT_DIR}/..:${RAFT_ROOT}:${ETCD_ROOT_DIR}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" \
+    run protoc --gofast_out=plugins=grpc:. -I=".:${GOGOPROTO_PATH}:${ETCD_ROOT_DIR}/..:${RAFT_ROOT}:${ETCD_ROOT_DIR}:${GOOGLEAPI_ROOT}" \
+      --gofast_opt=paths=source_relative,Mraftpb/raft.proto=go.etcd.io/raft/v3/raftpb,Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor \
       -I"${GRPC_GATEWAY_ROOT}" \
       --plugin="${GOFAST_BIN}" ./**/*.proto
 
-    run sed -i.bak -E 's|"etcd/api/|"go.etcd.io/etcd/api/v3/|g' ./**/*.pb.go
-    run sed -i.bak -E 's|"raftpb"|"go.etcd.io/raft/v3/raftpb"|g' ./**/*.pb.go
-    run sed -i.bak -E 's|"google/protobuf"|"github.com/gogo/protobuf/protoc-gen-gogo/descriptor"|g' ./**/*.pb.go
-
-    rm -f ./**/*.bak
     run gofmt -s -w ./**/*.pb.go
     run_go_tool "golang.org/x/tools/cmd/goimports" -w ./**/*.pb.go
   run popd
@@ -64,14 +81,17 @@ rm -rf Documentation/dev-guide/apispec/swagger/*json
 for pb in api/etcdserverpb/rpc server/etcdserver/api/v3lock/v3lockpb/v3lock server/etcdserver/api/v3election/v3electionpb/v3election; do
   log_callout "grpc & swagger for: ${pb}.proto"
   run protoc -I. \
-      -I"${GRPC_GATEWAY_ROOT}"/third_party/googleapis \
+      -I"${GOOGLEAPI_ROOT}" \
       -I"${GRPC_GATEWAY_ROOT}" \
       -I"${GOGOPROTO_PATH}" \
       -I"${ETCD_ROOT_DIR}/.." \
       -I"${RAFT_ROOT}" \
       --grpc-gateway_out=logtostderr=true,paths=source_relative:. \
-      --swagger_out=logtostderr=true:./Documentation/dev-guide/apispec/swagger/. \
-      --plugin="${SWAGGER_BIN}" --plugin="${GRPC_GATEWAY_BIN}" \
+      --grpc-gateway_opt=Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types \
+      --openapiv2_out=json_names_for_fields=false,logtostderr=true:./Documentation/dev-guide/apispec/swagger/. \
+      --openapiv2_opt=Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types:. \
+      --plugin="${OPENAPIV2_BIN}" \
+      --plugin="${GRPC_GATEWAY_BIN}" \
       ${pb}.proto
   # hack to move gw files around so client won't include them
   pkgpath=$(dirname "${pb}")

server/etcdserver/api/snap/snappb/snap.proto

@@ -3,6 +3,8 @@ package snappb;
 
 import "gogoproto/gogo.proto";
 
+option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/snap/snappb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

server/etcdserver/api/v3election/v3electionpb/v3election.proto

@@ -8,6 +8,8 @@ import "etcd/api/mvccpb/kv.proto";
 // for grpc-gateway
 import "google/api/annotations.proto";
 
+option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.unmarshaler_all) = true;
 

server/etcdserver/api/v3lock/v3lockpb/v3lock.proto

@@ -7,6 +7,8 @@ import "etcd/api/etcdserverpb/rpc.proto";
 // for grpc-gateway
 import "google/api/annotations.proto";
 
+option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.unmarshaler_all) = true;
 

server/lease/leasepb/lease.proto

@@ -4,6 +4,8 @@ package leasepb;
 import "gogoproto/gogo.proto";
 import "etcd/api/etcdserverpb/rpc.proto";
 
+option go_package = "go.etcd.io/etcd/server/v3/lease/leasepb;leasepb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

server/storage/wal/walpb/record.proto

@@ -4,6 +4,8 @@ package walpb;
 import "gogoproto/gogo.proto";
 import "raftpb/raft.proto";
 
+option go_package = "go.etcd.io/etcd/server/v3/storage/wal/walpb";
+
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.unmarshaler_all) = true;

tools/mod/go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/google/addlicense v1.1.1
 	github.com/google/yamlfmt v0.10.0
 	github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.17.1
 	github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f
 	github.com/mgechev/revive v1.3.3
 	github.com/mikefarah/yq/v4 v4.35.1
@@ -40,12 +40,11 @@ require (
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/goccy/go-yaml v1.11.0 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
@@ -85,10 +84,12 @@ require (
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
-	google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 // indirect
+	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/grpc v1.57.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect
 )