From 826050adf25496d7b3731ce19318df0876ba5c0e Mon Sep 17 00:00:00 2001 From: Mildred Ki'Lya Date: Sat, 13 Feb 2016 21:45:40 +0100 Subject: [PATCH] fix protocol buffer codec --- coding/coding.go | 26 ++---- coding/coding_test.go | 60 +++++++------- coding/pb/pbcodec.go | 182 +++++++++++++++++++++++++++--------------- 3 files changed, 155 insertions(+), 113 deletions(-) diff --git a/coding/coding.go b/coding/coding.go index 66eb3b2..620c812 100644 --- a/coding/coding.go +++ b/coding/coding.go @@ -7,19 +7,17 @@ import ( cbor "github.com/ipfs/go-ipld/coding/cbor" json "github.com/ipfs/go-ipld/coding/json" - mc "github.com/jbenet/go-multicodec" - mcproto "github.com/jbenet/go-multicodec/protobuf" - pb "github.com/ipfs/go-ipld/coding/pb" + memory "github.com/ipfs/go-ipld/memory" stream "github.com/ipfs/go-ipld/stream" + mc "github.com/jbenet/go-multicodec" ) var Header []byte const ( - HeaderPath = "/mdagv1" - ProtobufPath = "/protobuf/msgio" + HeaderPath = "/mdagv1" ) type Codec int @@ -47,8 +45,8 @@ func init() { cbor.HeaderWithTagsPath: func(r io.ReadSeeker) (stream.NodeReader, error) { return cbor.NewCBORDecoder(r) }, - ProtobufPath: func(r io.ReadSeeker) (stream.NodeReader, error) { - return DecodeLegacyProtobuf(r) + pb.MsgIOHeaderPath: func(r io.ReadSeeker) (stream.NodeReader, error) { + return pb.Decode(mc.WrapHeaderReader(pb.MsgIOHeader, r)) }, } } @@ -57,7 +55,7 @@ func Decode(r io.ReadSeeker) (stream.NodeReader, error) { // get multicodec first header, should be mcmux.Header err := mc.ConsumeHeader(r, Header) if err != nil { - return DecodeLegacyProtobuf(r) + return nil, err } // get next header, to select codec @@ -79,16 +77,8 @@ func DecodeBytes(data []byte) (stream.NodeReader, error) { return Decode(bytes.NewReader(data)) } -func DecodeLegacyProtobuf(r io.Reader) (stream.NodeReader, error) { - var node memory.Node = memory.Node{} - r = mc.WrapHeaderReader(mcproto.HeaderMsgio, r) - r = mc.WrapHeaderReader(pb.Header, r) - err := pb.Multicodec().Decoder(r).Decode(&node) - return node, err -} - func DecodeLegacyProtobufBytes(data []byte) (stream.NodeReader, error) { - return DecodeLegacyProtobuf(bytes.NewReader(data)) + return pb.RawDecode(data) } func EncodeRaw(codec Codec, w io.Writer, node memory.Node) error { @@ -100,7 +90,7 @@ func EncodeRaw(codec Codec, w io.Writer, node memory.Node) error { case CodecJSON: return json.Encode(w, node) case CodecProtobuf: - return fmt.Errorf("Protocol Buffer codec is not writeable") + return pb.Encode(w, node, true) default: return fmt.Errorf("Unknown codec %v", codec) } diff --git a/coding/coding_test.go b/coding/coding_test.go index 5ee63c0..f19bbfd 100644 --- a/coding/coding_test.go +++ b/coding/coding_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "testing" + links "github.com/ipfs/go-ipld/links" memory "github.com/ipfs/go-ipld/memory" reader "github.com/ipfs/go-ipld/stream" readertest "github.com/ipfs/go-ipld/test" @@ -37,8 +38,7 @@ type TC struct { // Test decoding and encoding a json and cbor file func TestCodecsEncodeDecode(t *testing.T) { - for _, fname := range []string{"json.testfile", "cbor.testfile"} { - testfile := codedFiles[fname] + for fname, testfile := range codedFiles { r, err := DecodeBytes(testfile) if err != nil { @@ -52,16 +52,22 @@ func TestCodecsEncodeDecode(t *testing.T) { codec = CodecJSON case "cbor.testfile": codec = CodecCBOR + case "protobuf.testfile": + codec = CodecProtobuf default: panic("should not arrive here") } + t.Logf("Decoded %s: %#v", fname, r) + n, err := memory.NewNodeFrom(r) if err != nil { t.Error(err) continue } + t.Logf("In memory %s: %#v", fname, n) + outData, err := EncodeBytes(codec, n) if err != nil { t.Error(err) @@ -120,6 +126,7 @@ func TestCborStream(t *testing.T) { func TestPbStream(t *testing.T) { a := assrt.NewAssert(t) t.Logf("Reading protobuf.testfile") + t.Logf("Bytes: %v", codedFiles["protobuf.testfile"]) pb, err := Decode(bytes.NewReader(codedFiles["protobuf.testfile"])) a.MustNil(err) @@ -127,34 +134,27 @@ func TestPbStream(t *testing.T) { readertest.Callback{[]interface{}{}, reader.TokenNode, nil}, readertest.Callback{[]interface{}{}, reader.TokenKey, "data"}, readertest.Callback{[]interface{}{"data"}, reader.TokenValue, []byte{0x08, 0x01}}, - readertest.Callback{[]interface{}{}, reader.TokenKey, "named-links"}, - readertest.Callback{[]interface{}{"named-links"}, reader.TokenNode, nil}, - readertest.Callback{[]interface{}{"named-links"}, reader.TokenKey, "a"}, - readertest.Callback{[]interface{}{"named-links", "a"}, reader.TokenNode, nil}, - readertest.Callback{[]interface{}{"named-links", "a"}, reader.TokenKey, "link"}, - readertest.Callback{[]interface{}{"named-links", "a", "link"}, reader.TokenValue, "Qmbvkmk9LFsGneteXk3G7YLqtLVME566ho6ibaQZZVHaC9"}, - readertest.Callback{[]interface{}{"named-links", "a"}, reader.TokenKey, "name"}, - readertest.Callback{[]interface{}{"named-links", "a", "name"}, reader.TokenValue, "a"}, - readertest.Callback{[]interface{}{"named-links", "a"}, reader.TokenKey, "size"}, - readertest.Callback{[]interface{}{"named-links", "a", "size"}, reader.TokenValue, uint64(10)}, - readertest.Callback{[]interface{}{"named-links", "a"}, reader.TokenEndNode, nil}, - readertest.Callback{[]interface{}{"named-links"}, reader.TokenKey, "b"}, - readertest.Callback{[]interface{}{"named-links", "b"}, reader.TokenNode, nil}, - readertest.Callback{[]interface{}{"named-links", "b"}, reader.TokenKey, "link"}, - readertest.Callback{[]interface{}{"named-links", "b", "link"}, reader.TokenValue, "QmR9pC5uCF3UExca8RSrCVL8eKv7nHMpATzbEQkAHpXmVM"}, - readertest.Callback{[]interface{}{"named-links", "b"}, reader.TokenKey, "name"}, - readertest.Callback{[]interface{}{"named-links", "b", "name"}, reader.TokenValue, "b"}, - readertest.Callback{[]interface{}{"named-links", "b"}, reader.TokenKey, "size"}, - readertest.Callback{[]interface{}{"named-links", "b", "size"}, reader.TokenValue, uint64(10)}, - readertest.Callback{[]interface{}{"named-links", "b"}, reader.TokenEndNode, nil}, - readertest.Callback{[]interface{}{"named-links"}, reader.TokenEndNode, nil}, - readertest.Callback{[]interface{}{}, reader.TokenKey, "ordered-links"}, - readertest.Callback{[]interface{}{"ordered-links"}, reader.TokenArray, nil}, - readertest.Callback{[]interface{}{"ordered-links"}, reader.TokenIndex, 0}, - readertest.Callback{[]interface{}{"ordered-links", 0}, reader.TokenValue, "a"}, - readertest.Callback{[]interface{}{"ordered-links"}, reader.TokenIndex, 1}, - readertest.Callback{[]interface{}{"ordered-links", 1}, reader.TokenValue, "b"}, - readertest.Callback{[]interface{}{"ordered-links"}, reader.TokenEndArray, nil}, + readertest.Callback{[]interface{}{}, reader.TokenKey, "links"}, + readertest.Callback{[]interface{}{"links"}, reader.TokenArray, nil}, + readertest.Callback{[]interface{}{"links"}, reader.TokenIndex, 0}, + readertest.Callback{[]interface{}{"links", 0}, reader.TokenNode, nil}, + readertest.Callback{[]interface{}{"links", 0}, reader.TokenKey, links.LinkKey}, + readertest.Callback{[]interface{}{"links", 0, links.LinkKey}, reader.TokenValue, "Qmbvkmk9LFsGneteXk3G7YLqtLVME566ho6ibaQZZVHaC9"}, + readertest.Callback{[]interface{}{"links", 0}, reader.TokenKey, "name"}, + readertest.Callback{[]interface{}{"links", 0, "name"}, reader.TokenValue, "a"}, + readertest.Callback{[]interface{}{"links", 0}, reader.TokenKey, "size"}, + readertest.Callback{[]interface{}{"links", 0, "size"}, reader.TokenValue, uint64(10)}, + readertest.Callback{[]interface{}{"links", 0}, reader.TokenEndNode, nil}, + readertest.Callback{[]interface{}{"links"}, reader.TokenIndex, 1}, + readertest.Callback{[]interface{}{"links", 1}, reader.TokenNode, nil}, + readertest.Callback{[]interface{}{"links", 1}, reader.TokenKey, links.LinkKey}, + readertest.Callback{[]interface{}{"links", 1, links.LinkKey}, reader.TokenValue, "QmR9pC5uCF3UExca8RSrCVL8eKv7nHMpATzbEQkAHpXmVM"}, + readertest.Callback{[]interface{}{"links", 1}, reader.TokenKey, "name"}, + readertest.Callback{[]interface{}{"links", 1, "name"}, reader.TokenValue, "b"}, + readertest.Callback{[]interface{}{"links", 1}, reader.TokenKey, "size"}, + readertest.Callback{[]interface{}{"links", 1, "size"}, reader.TokenValue, uint64(10)}, + readertest.Callback{[]interface{}{"links", 1}, reader.TokenEndNode, nil}, + readertest.Callback{[]interface{}{"links"}, reader.TokenEndArray, nil}, readertest.Callback{[]interface{}{}, reader.TokenEndNode, nil}, }) } diff --git a/coding/pb/pbcodec.go b/coding/pb/pbcodec.go index 84247c2..a7e620e 100644 --- a/coding/pb/pbcodec.go +++ b/coding/pb/pbcodec.go @@ -5,14 +5,19 @@ import ( "fmt" "io" + links "github.com/ipfs/go-ipld/links" memory "github.com/ipfs/go-ipld/memory" base58 "github.com/jbenet/go-base58" + msgio "github.com/jbenet/go-msgio" mc "github.com/jbenet/go-multicodec" mcproto "github.com/jbenet/go-multicodec/protobuf" ) +const HeaderPath = "/mdagv1" +const MsgIOHeaderPath = "/protobuf/msgio" + var Header []byte -var HeaderPath string +var MsgIOHeader []byte var ( errInvalidData = fmt.Errorf("invalid merkledag v1 protobuf, Data not bytes") @@ -20,8 +25,8 @@ var ( ) func init() { - HeaderPath = "/mdagv1" Header = mc.Header([]byte(HeaderPath)) + MsgIOHeader = mc.Header([]byte(MsgIOHeaderPath)) } type codec struct { @@ -67,7 +72,7 @@ func (c *encoder) Encode(v interface{}) error { return err } - n, err := ld2pbNode(nv) + n, err := ld2pbNode(nv, true) if err != nil { return err } @@ -94,78 +99,114 @@ func (c *decoder) Decode(v interface{}) error { return nil } -func ld2pbNode(in *memory.Node) (*PBNode, error) { +func Decode(r io.Reader) (memory.Node, error) { + err := mc.ConsumeHeader(r, MsgIOHeader) + if err != nil { + return nil, err + } + + length, err := msgio.ReadLen(r, nil) + if err != nil { + return nil, err + } + + data := make([]byte, length) + _, err = io.ReadFull(r, data) + if err != nil { + return nil, err + } + + return RawDecode(data) +} + +func RawDecode(data []byte) (memory.Node, error) { + var pbn *PBNode = new(PBNode) + + err := pbn.Unmarshal(data) + if err != nil { + return nil, err + } + + n := make(memory.Node) + pb2ldNode(pbn, &n) + + return n, err +} + +func Encode(w io.Writer, n memory.Node, strict bool) error { + _, err := w.Write(MsgIOHeader) + if err != nil { + return err + } + + data, err := RawEncode(n, strict) + if err != nil { + return err + } + + msgio.WriteLen(w, len(data)) + _, err = w.Write(data) + return err +} + +func RawEncode(n memory.Node, strict bool) ([]byte, error) { + pbn, err := ld2pbNode(&n, strict) + if err != nil { + return nil, err + } + + return pbn.Marshal() +} + +func ld2pbNode(in *memory.Node, strict bool) (*PBNode, error) { n := *in var pbn PBNode - var ordered_links []interface{} - var named_links memory.Node + size := 0 if data, hasdata := n["data"]; hasdata { + size++ data, ok := data.([]byte) if !ok { - return nil, errInvalidData + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: data is of incorect type") } pbn.Data = data + } else if strict { + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: no data") } - if ordered_links_node, ok := n["ordered-links"]; ok { - var ok bool - ordered_links, ok = ordered_links_node.([]interface{}) - if !ok { - return nil, errInvalidData + if links, ok := n["links"].([]interface{}); ok && links != nil { + size++ + for _, link := range links { + l, ok := link.(memory.Node) + if !ok { + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: link is of incorect type") + } + pblink, err := ld2pbLink(l, strict) + if err != nil { + return nil, err + } + pbn.Links = append(pbn.Links, pblink) } - } else { - return nil, errInvalidData + } else if strict { + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: no links") } - if named_links_node, ok := n["named-links"]; ok { - var ok bool - named_links, ok = named_links_node.(memory.Node) - if !ok { - return nil, errInvalidData - } - } else { - return nil, errInvalidData - } - - for n, link := range ordered_links { - var pblink *PBLink - var linkname string - switch link.(type) { - case string: - linkname = link.(string) - linknode := named_links[linkname].(memory.Node) - if linknode != nil { - pblink = ld2pbLink(linknode) - } - case memory.Node: - pblink = ld2pbLink(link.(memory.Node)) - linkname = link.(memory.Node)["name"].(string) - } - if pblink == nil { - return nil, fmt.Errorf("%s (#%d %s)", errInvalidLink, n, linkname) - } - pbn.Links = append(pbn.Links, pblink) + if strict && len(n) != size { + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: node contains extra fields (%d)", len(n)-size) } + return &pbn, nil } func pb2ldNode(pbn *PBNode, in *memory.Node) { var ordered_links []interface{} - var named_links memory.Node = make(memory.Node) for _, link := range pbn.Links { - if _, exists := named_links[link.GetName()]; exists { - ordered_links = append(ordered_links, pb2ldLink(link)) - } else { - ordered_links = append(ordered_links, link.GetName()) - named_links[link.GetName()] = pb2ldLink(link) - } + ordered_links = append(ordered_links, pb2ldLink(link)) } (*in)["data"] = pbn.GetData() - (*in)["named-links"] = named_links - (*in)["ordered-links"] = ordered_links + (*in)["links"] = ordered_links } func pb2ldLink(pbl *PBLink) (link memory.Node) { @@ -176,26 +217,37 @@ func pb2ldLink(pbl *PBLink) (link memory.Node) { }() link = make(memory.Node) - link["link"] = base58.Encode(pbl.Hash) + link[links.LinkKey] = base58.Encode(pbl.Hash) link["name"] = *pbl.Name link["size"] = uint64(*pbl.Tsize) return link } -func ld2pbLink(link memory.Node) (pbl *PBLink) { - defer func() { - if recover() != nil { - pbl = nil +func ld2pbLink(link memory.Node, strict bool) (pbl *PBLink, err error) { + length := 0 + pbl = &PBLink{} + + if hash, ok := link[links.LinkKey].(string); ok { + length++ + pbl.Hash = base58.Decode(hash) + if strict && base58.Encode(pbl.Hash) != hash { + return nil, errInvalidLink } - }() + } - hash := base58.Decode(link["hash"].(string)) - name := link["name"].(string) - size := link["size"].(uint64) + if name, ok := link["name"].(string); ok { + length++ + pbl.Name = &name + } - pbl = &PBLink{} - pbl.Hash = hash - pbl.Name = &name - pbl.Tsize = &size - return pbl + if size, ok := link["size"].(uint64); ok { + length++ + pbl.Tsize = &size + } + + if strict && len(link) != length { + return nil, fmt.Errorf("Invalid merkledag v1 protobuf: link contains %d fields instead of %d", len(link), length) + } + + return pbl, err }