Rework codec protocol buffer package

- add protocol buffer reader (and use the /mdagv1 multicodec prefix)
- regenerate protocol buffer code
- make the codec package stream oriented
- remove use of the Multicodec type as it is buffer oriented
- general cleanup

coding/Makefile

@@ -1,19 +1,34 @@
 
-pb/bin/multicodec:
-	$(MAKE) -C pb bin/multicodec
+bin/multicodec:
+	mkdir -p bin
+	go get -d github.com/jbenet/go-multicodec/multicodec
+	go build -o "$@" github.com/jbenet/go-multicodec/multicodec
 
 bin/convert:
 	cd bin; go build convert.go
 
-json.testfile: pb/bin/multicodec Makefile
+bin/msgio:
+	mkdir -p bin
+	go get -d github.com/jbenet/go-msgio/msgio
+	go build -o "$@" github.com/jbenet/go-msgio/msgio
+
+json.testfile: bin/multicodec Makefile
 	: >$@
-	pb/bin/multicodec header /multicodec >>$@
-	pb/bin/multicodec header /json >>$@
+	bin/multicodec header /multicodec >>$@
+	bin/multicodec header /json >>$@
 	echo '{"@codec":"/json","abc":{"mlink":"QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V"}}' >>$@
 
-cbor.testfile: json.testfile
-	pb/bin/multicodec header /multicodec >$@
+cbor.testfile: bin/multicodec json.testfile
+	bin/multicodec header /multicodec >$@
 	./convert -i $< -o $@.tmp -c '/cbor'
 	cat $@.tmp >>$@
 	rm -f $@.tmp
 
+protobuf.testfile: bin/multicodec bin/msgio
+	bin/multicodec header /mdagv1 >$@
+	bin/multicodec header /protobuf/msgio >>$@
+	mkdir -p dir
+	echo a >dir/a
+	echo b >dir/b
+	hash=`ipfs add -q -r dir | tail -n1` && \
+		ipfs object get "$$hash" --enc=protobuf | bin/msgio wrap >>$@

coding/coding.go

@@ -1,76 +1,101 @@
 package coding
 
 import (
-	mc "github.com/jbenet/go-multicodec"
-	mccbor "github.com/jbenet/go-multicodec/cbor"
-	mcmux "github.com/jbenet/go-multicodec/mux"
+	"bytes"
+	"fmt"
+	"io"
 
-	ipld "github.com/ipfs/go-ipld"
 	pb "github.com/ipfs/go-ipld/coding/pb"
-	memory "github.com/ipfs/go-ipld/memory"
+
+	ipld "github.com/ipfs/go-ipld"
+	stream "github.com/ipfs/go-ipld/coding/stream"
+	mc "github.com/jbenet/go-multicodec"
+)
+
+var Header []byte
+
+const (
+	HeaderPath = "/mdagv1"
 )
 
-// defaultCodec is the default applied if user does not specify a codec.
-// Most new objects will never specify a codec. We track the codecs with
-// the object so that multiple people using the same object will continue
-// to marshal using the same codec. the only reason this is important is
-// that the hashes must be the same.
-var defaultCodec string
+var StreamCodecs map[string]func(io.Reader) (stream.NodeReader, error)
 
-var muxCodec *mcmux.Multicodec
+type Codec int
+
+const (
+	NoCodec       Codec = 0
+	CodecProtobuf Codec = iota
+)
 
 func init() {
-	// by default, always encode things as cbor
-	defaultCodec = string(mc.HeaderPath(mccbor.Header))
-	muxCodec = mcmux.MuxMulticodec([]mc.Multicodec{
-		CborMulticodec(),
-		JsonMulticodec(),
-		pb.Multicodec(),
-	}, selectCodec)
-}
+	Header = mc.Header([]byte(HeaderPath))
 
-// Multicodec returns a muxing codec that marshals to
-// whatever codec makes sense depending on what information
-// the IPLD object itself carries
-func Multicodec() mc.Multicodec {
-	return muxCodec
+	StreamCodecs = map[string]func(io.Reader) (stream.NodeReader, error){
+		pb.MsgIOHeaderPath: func(r io.Reader) (stream.NodeReader, error) {
+			return pb.Decode(mc.WrapHeaderReader(pb.MsgIOHeader, r))
+		},
+	}
 }
 
-func selectCodec(v interface{}, codecs []mc.Multicodec) mc.Multicodec {
-	vn, ok := v.(*memory.Node)
-	if !ok {
-		return nil
+func DecodeReader(r io.Reader) (stream.NodeReader, error) {
+	// get multicodec first header, should be mcmux.Header
+	err := mc.ConsumeHeader(r, Header)
+	if err != nil {
+		return nil, err
 	}
 
-	codecKey, err := codecKey(*vn)
+	// get next header, to select codec
+	hdr, err := mc.ReadHeader(r)
 	if err != nil {
-		return nil
+		return nil, err
 	}
 
-	for _, c := range codecs {
-		if codecKey == string(mc.HeaderPath(c.Header())) {
-			return c
-		}
-	}
+	hdrPath := string(mc.HeaderPath(hdr))
 
-	return nil // no codec
+	fun, ok := StreamCodecs[hdrPath]
+	if !ok {
+		return nil, fmt.Errorf("no codec for %s", hdrPath)
+	}
+	return fun(r)
 }
 
-func codecKey(n memory.Node) (string, error) {
-	chdr, ok := (n)[ipld.CodecKey]
-	if !ok {
-		// if no codec is defined, use our default codec
-		chdr = defaultCodec
-		if pb.IsOldProtobufNode(n) {
-			chdr = string(pb.Header)
-		}
+func Decode(r io.Reader) (interface{}, error) {
+	rd, err := DecodeReader(r)
+	if err != nil {
+		return nil, err
 	}
 
-	chdrs, ok := chdr.(string)
-	if !ok {
-		// if chdr is not a string, cannot read codec.
-		return "", mc.ErrType
+	return stream.NewNodeFromReader(rd)
+}
+
+func DecodeBytes(data []byte) (interface{}, error) {
+	return Decode(bytes.NewReader(data))
+}
+
+func HasHeader(data []byte) bool {
+	return len(data) >= len(Header) && bytes.Equal(data[:len(Header)], Header)
+}
+
+func DecodeLegacyProtobufBytes(data []byte) (stream.NodeReader, error) {
+	return pb.RawDecode(data)
+}
+
+func EncodeRaw(codec Codec, w io.Writer, node ipld.NodeIterator) error {
+	switch codec {
+	case CodecProtobuf:
+		return pb.Encode(w, node, true)
+	default:
+		return fmt.Errorf("Unknown codec %v", codec)
 	}
+}
+
+func Encode(codec Codec, w io.Writer, node ipld.NodeIterator) error {
+	w.Write(Header)
+	return EncodeRaw(codec, w, node)
+}
 
-	return chdrs, nil
+func EncodeBytes(codec Codec, node ipld.NodeIterator) ([]byte, error) {
+	var buf bytes.Buffer
+	err := Encode(codec, &buf, node)
+	return buf.Bytes(), err
 }