diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..0277e19 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "**/*.pb.go" # ignore protobuf generated code diff --git a/go.sum b/go.sum index b317b76..5a11490 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/bradleyjkemp/cupaloy/v2 v2.5.0 h1:XI37Pqyl+msFaJDYL3JuPFKGUgnVxyJp+gQZQGiz2nA= github.com/bradleyjkemp/cupaloy/v2 v2.5.0/go.mod h1:TD5UU0rdYTbu/TtuwFuWrtiRARuN7mtRipvs/bsShSE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= @@ -25,8 +26,6 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/improbable-eng/grpc-web v0.9.6 h1:B8FH/k5xv/vHovSt70GJHIB2/1+4plmvtfrz33ambuE= github.com/improbable-eng/grpc-web v0.9.6/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= -github.com/jhump/protoreflect v1.4.3 h1:eUSvsY6mTZSsniEXgoFTO5vF2I5VSifUNvtKotc7cl8= -github.com/jhump/protoreflect v1.4.3/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8= github.com/jhump/protoreflect v1.4.4 h1:kySdALZUh7xRtW6UoZjjHtlR8k7rLzx5EXJFRvsO5UY= github.com/jhump/protoreflect v1.4.4/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/grpc-dump/dump/dump.go b/grpc-dump/dump/dump.go index cfbb002..da3e59f 100644 --- a/grpc-dump/dump/dump.go +++ b/grpc-dump/dump/dump.go @@ -23,8 +23,6 @@ func Run(output io.Writer, protoRoots, protoDescriptors string, proxyConfig ...g } resolvers = append(resolvers, r) } - // Always use the unknown message resolver - resolvers = append(resolvers, proto_decoder.NewUnknownResolver()) opts := append(proxyConfig, grpc_proxy.WithInterceptor(dumpInterceptor(output, proto_decoder.NewDecoder(resolvers...)))) proxy, err := grpc_proxy.New( diff --git a/grpc-fixture/fixture/fixture_interceptor.go b/grpc-fixture/fixture/fixture_interceptor.go index f649d0f..6662f54 100644 --- a/grpc-fixture/fixture/fixture_interceptor.go +++ b/grpc-fixture/fixture/fixture_interceptor.go @@ -55,7 +55,7 @@ func (f fixture) intercept(srv interface{}, ss grpc.ServerStream, info *grpc.Str } if !found { - return status.Error(codes.Unavailable, "no matching saved responses for method "+info.FullMethod) + return status.Errorf(codes.Unavailable, "no matching saved responses for method %s and message", info.FullMethod) } } diff --git a/integration_test/.snapshots/TestIntegration.json b/integration_test/.snapshots/TestIntegration.json index e9e20e9..9c6b892 100644 --- a/integration_test/.snapshots/TestIntegration.json +++ b/integration_test/.snapshots/TestIntegration.json @@ -1,6 +1,6 @@ {"service":"bradleyjkemp.github.io.TestService","method":"TestUnaryClientRequest","messages":[{"message_origin":"client","raw_message":"ChEaDUNsaWVudFJlcXVlc3QgARAB","message":{"outerValue":{"innerValue":"ClientRequest","innerNum":1},"outerNum":1},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlclJlc3BvbnNlIAIQAg==","message":{"outerValue":{"innerValue":"ServerResponse","innerNum":2},"outerNum":2},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["bradleyjkemp.github.io:444"],"content-type":["application/grpc"],"user-agent":["grpc-go/1.22.0"],"via":["HTTP/2.0 127.0.0.1:16354"]}} {"service":"bradleyjkemp.github.io.TestService","method":"TestUnaryClientRequest","messages":[{"message_origin":"client","raw_message":"ChEaDUNsaWVudFJlcXVlc3QgARAB","message":{"outerValue":{"innerValue":"ClientRequest","innerNum":1},"outerNum":1},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlclJlc3BvbnNlIAIQAg==","message":{"outerValue":{"innerValue":"ServerResponse","innerNum":2},"outerNum":2},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["bradleyjkemp.github.io:444"],"content-type":["application/grpc"],"user-agent":["grpc-go/1.22.0"],"via":["HTTP/2.0 127.0.0.1:16354"]}} -{"service":"bradleyjkemp.github.io.TestService","method":"TestStreamingServerMessages","messages":[{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"1":{"3":"ServerMessage1","4":3},"2":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UyIAQQBA==","message":{"1":{"3":"ServerMessage2","4":4},"2":4},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["a-different-domain.github.io:444"],"content-type":["application/grpc"],"forwarded":["proto=https"],"user-agent":["grpc-go/1.22.0"],"via":["HTTP/2.0 127.0.0.1:16354"]}} +{"service":"bradleyjkemp.github.io.TestService","method":"TestStreamingServerMessages","messages":[{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"outerValue":{"innerValue":"ServerMessage1","innerNum":3},"outerNum":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UyIAUQBRoORGV0ZWN0ZWQgdmFsdWU=","message":{"outerValue":{"innerValue":"ServerMessage2","innerNum":5},"outerNum":5,"3":"Detected value"},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["a-different-domain.github.io:444"],"content-type":["application/grpc"],"forwarded":["proto=https"],"user-agent":["grpc-go/1.22.0"],"via":["HTTP/2.0 127.0.0.1:16354"]}} {"service":"grpc.gateway.testing.EchoService","method":"Echo","messages":[{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"1":{"3":"ServerMessage1","4":3},"2":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"1":{"3":"ServerMessage1","4":3},"2":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["grpc-web.github.io"],"accept":["*/*"],"accept-encoding":["gzip, deflate, br"],"accept-language":["en-US,en;q=0.9"],"cache-control":["no-cache"],"content-type":["application/grpc+proto"],"custom-header-1":["value1"],"origin":["http://localhost:8081"],"pragma":["no-cache"],"referer":["http://localhost:8081/echotest.html"],"user-agent":["Mozilla/5.0"],"via":["HTTP/2.0 127.0.0.1:16354"],"x-grpc-web":["1"],"x-user-agent":["grpc-web-javascript/0.1"]}} {"service":"grpc.gateway.testing.EchoService","method":"Echo","messages":[{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"1":{"3":"ServerMessage1","4":3},"2":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"},{"message_origin":"server","raw_message":"ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==","message":{"1":{"3":"ServerMessage1","4":3},"2":3},"timestamp":"2019-06-24T19:19:46.644943+01:00"}],"metadata":{":authority":["grpc-web.github.io:1234"],"accept":["*/*"],"accept-encoding":["gzip, deflate, br"],"accept-language":["en-US,en;q=0.9"],"cache-control":["no-cache"],"content-type":["application/grpc+proto"],"custom-header-1":["value1"],"forwarded":["proto=https"],"origin":["http://localhost:8081"],"pragma":["no-cache"],"referer":["http://localhost:8081/echotest.html"],"user-agent":["Mozilla/5.0"],"via":["HTTP/2.0 127.0.0.1:16354"],"x-grpc-web":["1"],"x-user-agent":["grpc-web-javascript/0.1"]}} diff --git a/integration_test/generate.go b/integration_test/generate.go index c33421c..a2785f8 100644 --- a/integration_test/generate.go +++ b/integration_test/generate.go @@ -24,4 +24,15 @@ func main() { fmt.Println(i, val, base64.StdEncoding.EncodeToString(marshalled)) } + + marshalled, _ := proto.Marshal(&OuterWithExtra{ + OuterValue: &Inner{ + InnerValue: "ServerMessage2", + InnerNum: int64(4 + 1), + }, + OuterNum: int64(4 + 1), + ExtraField: "Detected value", + }) + + fmt.Println(4, "ServerMessage2", base64.StdEncoding.EncodeToString(marshalled)) } diff --git a/integration_test/integration_test.go b/integration_test/integration_test.go index 0f5c308..bba24d3 100644 --- a/integration_test/integration_test.go +++ b/integration_test/integration_test.go @@ -37,7 +37,8 @@ func TestIntegration(t *testing.T) { defer func() { select { case err := <-errors: - t.Fatal("Unexpected error:", err) + t.Log("Unexpected error:", err) + t.Fail() default: return } @@ -47,7 +48,7 @@ func TestIntegration(t *testing.T) { fixtureErr := fixture.Run( protoRoots, protoDescriptors, - ".snapshots/TestIntegration.json", + "test-fixture.json", grpc_proxy.Port(fixturePort), grpc_proxy.UsingTLS(certFile, keyFile), ) @@ -87,17 +88,20 @@ func TestIntegration(t *testing.T) { }), ) if replayErr != nil { - t.Fatal("Unexpected error:", replayErr) + t.Log("Unexpected error:", replayErr) + t.Fail() } cmd := curlCommand("http://grpc-web.github.io/grpc.gateway.testing.EchoService/Echo") if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal("Unexpected error:", err, string(out)) + t.Log("Unexpected error:", err, string(out)) + t.Fail() } cmd = curlCommand("https://grpc-web.github.io:1234/grpc.gateway.testing.EchoService/Echo") if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal("Unexpected error:", err, string(out)) + t.Log("Unexpected error:", err, string(out)) + t.Fail() } dumpLogSanitised := timestampRegex.ReplaceAll(dumpLog.Bytes(), []byte("\"timestamp\":\"2019-06-24T19:19:46.644943+01:00\"")) diff --git a/integration_test/test-dump.json b/integration_test/test-dump.json index a3ad9ca..8f6495e 100644 --- a/integration_test/test-dump.json +++ b/integration_test/test-dump.json @@ -65,7 +65,7 @@ }, { "message_origin": "server", - "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UyIAQQBA==" + "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UyIAUQBRoORGV0ZWN0ZWQgdmFsdWU=" } ], "metadata": { diff --git a/integration_test/test-fixture.json b/integration_test/test-fixture.json new file mode 100644 index 0000000..15bc9b0 --- /dev/null +++ b/integration_test/test-fixture.json @@ -0,0 +1,83 @@ +{ + "service": "bradleyjkemp.github.io.TestService", + "method": "TestUnaryClientRequest", + "messages": [ + { + "message_origin": "client", + "raw_message": "ChEaDUNsaWVudFJlcXVlc3QgARAB", + "message": { + "outerValue": { + "innerValue": "ClientRequest", + "innerNum": 1 + }, + "outerNum": 1 + }, + "timestamp": "2019-06-24T19:19:46.644943+01:00" + }, + { + "message_origin": "server", + "raw_message": "ChIaDlNlcnZlclJlc3BvbnNlIAIQAg==", + "message": { + "outerValue": { + "innerValue": "ServerResponse", + "innerNum": 2 + }, + "outerNum": 2 + }, + "timestamp": "2019-06-24T19:19:46.644943+01:00" + } + ] +} +{ + "service": "bradleyjkemp.github.io.TestService", + "method": "TestStreamingServerMessages", + "messages": [ + { + "message_origin": "server", + "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==", + "message": { + "1": { + "3": "ServerMessage1", + "4": 3 + }, + "2": 3 + }, + "timestamp": "2019-06-24T19:19:46.644943+01:00" + }, + { + "message_origin": "server", + "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UyIAUQBRoORGV0ZWN0ZWQgdmFsdWU=", + "timestamp": "2019-06-24T19:19:46.644943+01:00" + } + ] +} +{ + "service": "grpc.gateway.testing.EchoService", + "method": "Echo", + "messages": [ + { + "message_origin": "server", + "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==", + "message": { + "1": { + "3": "ServerMessage1", + "4": 3 + }, + "2": 3 + }, + "timestamp": "2019-06-24T19:19:46.644943+01:00" + }, + { + "message_origin": "server", + "raw_message": "ChIaDlNlcnZlck1lc3NhZ2UxIAMQAw==", + "message": { + "1": { + "3": "ServerMessage1", + "4": 3 + }, + "2": 3 + }, + "timestamp": "2019-06-24T19:19:46.644943+01:00" + } + ] +} diff --git a/integration_test/test.pb.go b/integration_test/test.pb.go index 451faf1..97f85d6 100644 --- a/integration_test/test.pb.go +++ b/integration_test/test.pb.go @@ -18,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Outer struct { OuterValue *Inner `protobuf:"bytes,1,opt,name=outer_value,json=outerValue,proto3" json:"outer_value,omitempty"` @@ -114,23 +114,85 @@ func (m *Inner) GetInnerNum() int64 { return 0 } +type OuterWithExtra struct { + OuterValue *Inner `protobuf:"bytes,1,opt,name=outer_value,json=outerValue,proto3" json:"outer_value,omitempty"` + OuterNum int64 `protobuf:"varint,2,opt,name=outer_num,json=outerNum,proto3" json:"outer_num,omitempty"` + ExtraField string `protobuf:"bytes,3,opt,name=extra_field,json=extraField,proto3" json:"extra_field,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *OuterWithExtra) Reset() { *m = OuterWithExtra{} } +func (m *OuterWithExtra) String() string { return proto.CompactTextString(m) } +func (*OuterWithExtra) ProtoMessage() {} +func (*OuterWithExtra) Descriptor() ([]byte, []int) { + return fileDescriptor_c161fcfdc0c3ff1e, []int{2} +} + +func (m *OuterWithExtra) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_OuterWithExtra.Unmarshal(m, b) +} +func (m *OuterWithExtra) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_OuterWithExtra.Marshal(b, m, deterministic) +} +func (m *OuterWithExtra) XXX_Merge(src proto.Message) { + xxx_messageInfo_OuterWithExtra.Merge(m, src) +} +func (m *OuterWithExtra) XXX_Size() int { + return xxx_messageInfo_OuterWithExtra.Size(m) +} +func (m *OuterWithExtra) XXX_DiscardUnknown() { + xxx_messageInfo_OuterWithExtra.DiscardUnknown(m) +} + +var xxx_messageInfo_OuterWithExtra proto.InternalMessageInfo + +func (m *OuterWithExtra) GetOuterValue() *Inner { + if m != nil { + return m.OuterValue + } + return nil +} + +func (m *OuterWithExtra) GetOuterNum() int64 { + if m != nil { + return m.OuterNum + } + return 0 +} + +func (m *OuterWithExtra) GetExtraField() string { + if m != nil { + return m.ExtraField + } + return "" +} + func init() { - proto.RegisterType((*Outer)(nil), "main.Outer") - proto.RegisterType((*Inner)(nil), "main.Inner") + proto.RegisterType((*Outer)(nil), "bradleyjkemp.github.io.Outer") + proto.RegisterType((*Inner)(nil), "bradleyjkemp.github.io.Inner") + proto.RegisterType((*OuterWithExtra)(nil), "bradleyjkemp.github.io.OuterWithExtra") } func init() { proto.RegisterFile("test.proto", fileDescriptor_c161fcfdc0c3ff1e) } var fileDescriptor_c161fcfdc0c3ff1e = []byte{ - // 153 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0x49, 0x2d, 0x2e, - 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc, 0x53, 0x0a, 0xe2, 0x62, - 0xf5, 0x2f, 0x2d, 0x49, 0x2d, 0x12, 0xd2, 0xe1, 0xe2, 0xce, 0x07, 0x31, 0xe2, 0xcb, 0x12, 0x73, - 0x4a, 0x53, 0x25, 0x18, 0x15, 0x18, 0x35, 0xb8, 0x8d, 0xb8, 0xf5, 0x40, 0x8a, 0xf4, 0x3c, 0xf3, - 0xf2, 0x52, 0x8b, 0x82, 0xb8, 0xc0, 0xf2, 0x61, 0x20, 0x69, 0x21, 0x69, 0x2e, 0x4e, 0x88, 0xea, - 0xbc, 0xd2, 0x5c, 0x09, 0x26, 0x05, 0x46, 0x0d, 0xe6, 0x20, 0x0e, 0xb0, 0x80, 0x5f, 0x69, 0xae, - 0x92, 0x2b, 0x17, 0x2b, 0x58, 0x87, 0x90, 0x3c, 0x17, 0x77, 0x26, 0x88, 0x01, 0x35, 0x93, 0x59, - 0x81, 0x51, 0x83, 0x33, 0x88, 0x0b, 0x2c, 0x04, 0x37, 0x06, 0xa2, 0x00, 0x64, 0x0c, 0x0b, 0xc4, - 0x18, 0xb0, 0x80, 0x5f, 0x69, 0x6e, 0x12, 0x1b, 0xd8, 0x9d, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, - 0xff, 0xd0, 0xfc, 0x79, 0xb9, 0xb5, 0x00, 0x00, 0x00, + // 253 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x91, 0x3d, 0x4f, 0xc3, 0x30, + 0x10, 0x86, 0x31, 0xa5, 0x88, 0x5e, 0x24, 0x06, 0x0f, 0x55, 0x05, 0x42, 0x44, 0x99, 0x32, 0x79, + 0x28, 0x3b, 0x0b, 0x2a, 0x12, 0x0b, 0x48, 0xe1, 0x73, 0xab, 0x12, 0x72, 0xb4, 0x86, 0xc4, 0x2e, + 0xce, 0xb9, 0xa2, 0x7f, 0x82, 0xdf, 0x8c, 0xee, 0x02, 0x4c, 0xc0, 0xc6, 0x76, 0x79, 0xe3, 0xe7, + 0xbd, 0xc7, 0x32, 0x00, 0x61, 0x47, 0x66, 0x15, 0x3c, 0x79, 0x3d, 0xae, 0x42, 0x59, 0x37, 0xb8, + 0x79, 0x7e, 0xc1, 0x76, 0x65, 0x16, 0x96, 0x96, 0xb1, 0x32, 0xd6, 0x67, 0x35, 0x0c, 0xaf, 0x22, + 0x61, 0xd0, 0xa7, 0x90, 0x78, 0x1e, 0xe6, 0xeb, 0xb2, 0x89, 0x38, 0x51, 0xa9, 0xca, 0x93, 0xe9, + 0x91, 0xf9, 0x19, 0x33, 0x17, 0xce, 0x61, 0x28, 0x40, 0x88, 0x3b, 0x06, 0xf4, 0x21, 0x8c, 0x7a, + 0xde, 0xc5, 0x76, 0xb2, 0x9d, 0xaa, 0x7c, 0x50, 0xec, 0x49, 0x70, 0x19, 0xdb, 0x6c, 0x06, 0x43, + 0x21, 0xf4, 0x31, 0x24, 0x96, 0x87, 0xcf, 0x2d, 0x83, 0x54, 0xe5, 0xa3, 0x02, 0x24, 0xfa, 0xae, + 0xe9, 0x0f, 0x70, 0xcd, 0x4e, 0x5f, 0x23, 0x01, 0xd7, 0xbc, 0x2b, 0xd8, 0x17, 0xdb, 0x7b, 0x4b, + 0xcb, 0xd9, 0x1b, 0x85, 0xf2, 0x5f, 0xb5, 0xd9, 0x16, 0x79, 0xcb, 0xfc, 0xc9, 0x62, 0x53, 0x7f, + 0xd9, 0x4a, 0x74, 0xce, 0xc9, 0x74, 0x01, 0xc9, 0x0d, 0x76, 0x74, 0x8d, 0x61, 0x6d, 0x1f, 0x51, + 0x3f, 0xc0, 0x98, 0x3f, 0x6f, 0x5d, 0x19, 0x36, 0x67, 0x8d, 0x45, 0x47, 0x05, 0xbe, 0x46, 0xec, + 0x48, 0xff, 0x6a, 0x24, 0xd7, 0x39, 0xf8, 0xfb, 0x77, 0xb6, 0x55, 0xed, 0xca, 0x2b, 0x9e, 0x7c, + 0x04, 0x00, 0x00, 0xff, 0xff, 0xf2, 0xb4, 0x0b, 0xad, 0xd3, 0x01, 0x00, 0x00, } diff --git a/integration_test/test.proto b/integration_test/test.proto index 2102c9a..1798652 100644 --- a/integration_test/test.proto +++ b/integration_test/test.proto @@ -11,6 +11,13 @@ message Inner { int64 inner_num = 4; } +message OuterWithExtra { + Inner outer_value = 1; + int64 outer_num = 2; + string extra_field = 3; +} + service TestService { rpc TestUnaryClientRequest(Outer) returns (Outer) {}; + rpc TestStreamingServerMessages(Outer) returns (stream Outer) {}; } diff --git a/internal/proto_decoder/decode.go b/internal/proto_decoder/decode.go index af66132..4eaf4ec 100644 --- a/internal/proto_decoder/decode.go +++ b/internal/proto_decoder/decode.go @@ -22,7 +22,8 @@ type MessageDecoder interface { } type messageDecoder struct { - resolvers []MessageResolver + resolvers []MessageResolver + unknownField unknownFieldResolver } // Chain together a number of resolvers to decode incoming messages. @@ -30,7 +31,8 @@ type messageDecoder struct { // is used to decode the message. func NewDecoder(resolvers ...MessageResolver) *messageDecoder { return &messageDecoder{ - resolvers: resolvers, + resolvers: append(resolvers, emptyResolver{}), + unknownField: unknownFieldResolver{}, } } @@ -42,8 +44,13 @@ func (d *messageDecoder) Decode(fullMethod string, message *internal.Message) (* if err != nil { continue } + // check for any unknown fields and add them to the descriptor + descriptor, err := d.unknownField.enrichDecodeDescriptor(descriptor, message) + if err != nil { + continue + } dyn := dynamic.NewMessage(descriptor) - // now unmarshal again using the new generated message type + // now unmarshal using the enriched message type err = proto.Unmarshal(message.RawMessage, dyn) if err == nil { return dyn, nil diff --git a/internal/proto_decoder/encode.go b/internal/proto_decoder/encode.go index 2694991..03075d0 100644 --- a/internal/proto_decoder/encode.go +++ b/internal/proto_decoder/encode.go @@ -20,10 +20,12 @@ type MessageEncoder interface { // Chain together a number of resolvers to decode incoming messages. // Resolvers are in priority order, the first to return a nil error -// is used to decode the message. +// is used to decode the message. If no resolvers are successful, +// a default resolver is used that always returns empty.Empty func NewEncoder(resolvers ...MessageResolver) *messageEncoder { return &messageEncoder{ - resolvers: resolvers, + resolvers: append(resolvers), + // TODO: include an unknown message encoder here } } diff --git a/internal/proto_decoder/proto_resolver.go b/internal/proto_decoder/proto_resolver.go index 2e01c2b..e2163f5 100644 --- a/internal/proto_decoder/proto_resolver.go +++ b/internal/proto_decoder/proto_resolver.go @@ -4,7 +4,10 @@ import ( "fmt" "github.com/bradleyjkemp/grpc-tools/internal" "github.com/bradleyjkemp/grpc-tools/internal/proto_descriptor" + "github.com/golang/protobuf/ptypes/empty" "github.com/jhump/protoreflect/desc" + "github.com/jhump/protoreflect/desc/builder" + "strings" ) type descriptorResolver struct { @@ -52,3 +55,27 @@ func NewDescriptorResolver(protoFileDescriptors ...string) (*descriptorResolver, descs, }, nil } + +var messageName = strings.NewReplacer( + "/", "_", + ".", "_", +) + +type emptyResolver struct{} + +func (e emptyResolver) resolveEncoded(fullMethod string, message *internal.Message) (*desc.MessageDescriptor, error) { + d, err := desc.LoadMessageDescriptorForMessage(&empty.Empty{}) + if err != nil { + return nil, err + } + mb, err := builder.FromMessage(d) + if err != nil { + return nil, err + } + mb.SetName(fmt.Sprintf("%s_%s", messageName.Replace(fullMethod), message.MessageOrigin)) + return mb.Build() +} + +func (e emptyResolver) resolveDecoded(fullMethod string, message *internal.Message) (*desc.MessageDescriptor, error) { + return desc.LoadMessageDescriptorForMessage(&empty.Empty{}) +} diff --git a/internal/proto_decoder/unknown_message.go b/internal/proto_decoder/unknown_message.go index 1c703b9..6fd59cf 100644 --- a/internal/proto_decoder/unknown_message.go +++ b/internal/proto_decoder/unknown_message.go @@ -9,99 +9,121 @@ import ( "github.com/jhump/protoreflect/desc/builder" "github.com/jhump/protoreflect/dynamic" "regexp" - "sync/atomic" ) // When we don't have an actual proto message descriptor, this takes a best effort // approach to generating one. It's definitely not perfect but is more useful than nothing. -type unknownMessageResolver struct { - messageCounter int64 // must only be accessed atomically -} +type unknownFieldResolver struct{} -func NewUnknownResolver() *unknownMessageResolver { - return &unknownMessageResolver{ - messageCounter: 0, +// This takes a message descriptor and enriches it to add any unknown fields present. +// This means that all unknown fields will show up in the dump. +func (u *unknownFieldResolver) enrichDecodeDescriptor(resolved *desc.MessageDescriptor, message *internal.Message) (*desc.MessageDescriptor, error) { + decoded := dynamic.NewMessage(resolved) + err := proto.Unmarshal(message.RawMessage, decoded) + if err != nil { + return nil, err + } + descriptor, err := builder.FromMessage(resolved) + if err != nil { + return nil, err } + err = u.enrichMessage(descriptor, decoded) + if err != nil { + return nil, err + } + return descriptor.Build() } -func (u *unknownMessageResolver) resolveEncoded(fullMethod string, message *internal.Message) (*desc.MessageDescriptor, error) { - dyn, _ := dynamic.AsDynamicMessage(&empty.Empty{}) - err := proto.Unmarshal(message.RawMessage, dyn) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal bytes: %v", err) +func (u *unknownFieldResolver) enrichMessage(descriptor *builder.MessageBuilder, message *dynamic.Message) error { + for _, fieldNum := range message.GetUnknownFields() { + generatedFieldName := fmt.Sprintf("%s_%d", descriptor.GetName(), fieldNum) + unknownFieldContents := message.GetUnknownField(fieldNum) + fieldType, err := u.detectFieldType(generatedFieldName, unknownFieldContents) + if err != nil { + return err + } + field := builder.NewField(generatedFieldName, fieldType) + field.SetNumber(fieldNum) + field.SetJsonName(fmt.Sprintf("%d", fieldNum)) + if len(unknownFieldContents) > 1 { + field.SetRepeated() + } + if descriptor.TryAddField(field) != nil { + return err + } } - return u.generateDescriptorForUnknownMessage(dyn).Build() -} + // recurse into the known fields to check for nested unknown fields + for _, fieldDescriptor := range message.GetKnownFields() { + if fieldDescriptor.GetMessageType() != nil { + nestedMessage, ok := message.GetField(fieldDescriptor).(proto.Message) + if !ok { + return fmt.Errorf("error: nested message was not of type proto.Message") + } + nestedMessageDescriptor, err := builder.FromMessage(fieldDescriptor.GetMessageType()) + if err != nil { + return err + } + dynamicNestedMessage := dynamic.NewMessage(fieldDescriptor.GetMessageType()) + err = dynamicNestedMessage.MergeFrom(nestedMessage) + if err != nil { + return err + } + err = u.enrichMessage(nestedMessageDescriptor, dynamicNestedMessage) + if err != nil { + return err + } + fieldDescriptorBuilder, err := builder.FromField(fieldDescriptor) + if err != nil { + return err + } + fieldDescriptorBuilder.SetType(builder.FieldTypeMessage(nestedMessageDescriptor)) + descriptor.RemoveField(fieldDescriptor.GetName()).AddField(fieldDescriptorBuilder) + } + } -func (u *unknownMessageResolver) resolveDecoded(fullMethod string, message *internal.Message) (*desc.MessageDescriptor, error) { - return nil, fmt.Errorf("unimplemented") + return nil } var ( asciiPattern = regexp.MustCompile(`^[ -~]*$`) ) -// All messages must have unique names, this function generates them -func (u *unknownMessageResolver) getNewMessageName() string { - return fmt.Sprintf("unknown_message_%d", atomic.AddInt64(&u.messageCounter, 1)) -} - -// This takes a dynamic.Message of unknown type and returns -// a message descriptor which will mean all fields are included -// in the JSON output. -// TODO: this would probably be better implemented within github.com/jhump/protoreflect -func (u *unknownMessageResolver) generateDescriptorForUnknownMessage(message *dynamic.Message) *builder.MessageBuilder { - fields := map[int32][]dynamic.UnknownField{} - for _, unknownFieldNum := range message.GetUnknownFields() { - fields[unknownFieldNum] = message.GetUnknownField(unknownFieldNum) - } - return u.makeDescriptorForFields(fields) -} - -func (u *unknownMessageResolver) makeDescriptorForFields(fields map[int32][]dynamic.UnknownField) *builder.MessageBuilder { - msg := builder.NewMessage(u.getNewMessageName()) - for fieldNum, instances := range fields { - var fieldType *builder.FieldType - // TODO: look at all instances and merge the discovered fields together - // This way repeated sub messages are handled properly (i.e. where the full type information - // might not be inferable from just the first message) - switch instances[0].Encoding { - // TODO: handle all wire types - case proto.WireBytes: - fieldType = u.handleWireBytes(instances[0]) - default: - // Fixed precision number - fieldType = builder.FieldTypeFixed64() +func (u *unknownFieldResolver) detectFieldType(fieldName string, fields []dynamic.UnknownField) (*builder.FieldType, error) { + field := fields[0] + switch field.Encoding { + // TODO: handle all wire types + case proto.WireBytes: + if asciiPattern.Match(field.Contents) { + // highly unlikely that an entirely ASCII string is actually an embedded proto message + // TODO: make this heuristic cleverer + return builder.FieldTypeString(), nil } - field := builder.NewField(fmt.Sprintf("_%d", fieldNum), fieldType) - field.SetNumber(fieldNum) - if len(instances) > 1 { - field.SetRepeated() + // embedded messages are encoded on the wire as strings + // so try to decode this string as a message + dyn, err := dynamic.AsDynamicMessage(&empty.Empty{}) + if err != nil { + panic(err) } - msg.AddField(field) - } + err = proto.Unmarshal(field.Contents, dyn) + if err != nil { + // looks like it wasn't a valid proto message + return builder.FieldTypeString(), nil + } + // TODO: check that the unmarshalled message doesn't have any illegal field numbers - return msg -} + // probably is an embedded message + descriptor, _ := builder.FromMessage(dyn.GetMessageDescriptor()) + descriptor.SetName(fieldName) + err = u.enrichMessage(descriptor, dyn) + if err != nil { + return nil, err + } + return builder.FieldTypeMessage(descriptor), nil -func (u *unknownMessageResolver) handleWireBytes(instance dynamic.UnknownField) *builder.FieldType { - if asciiPattern.Match(instance.Contents) { - // highly unlikely that an entirely ASCII string is actually an embedded proto message - // TODO: make this heuristic cleverer - return builder.FieldTypeString() - } - // embedded messages are encoded on the wire as strings - // so try to decode this string as a message - dyn, err := dynamic.AsDynamicMessage(&empty.Empty{}) - if err != nil { - panic(err) - } - err = proto.Unmarshal(instance.Contents, dyn) - if err != nil { - // looks like it wasn't a valid proto message - return builder.FieldTypeString() + default: + // Fixed precision number + return builder.FieldTypeFixed64(), nil } - return builder.FieldTypeMessage(u.generateDescriptorForUnknownMessage(dyn)) }