Add CBOR & JSON codec implementation

- Decode to the NodeReader interface
- Add writeable support for JSON and CBOR from memory
- Implement CBOR link tag for reading and writing
- Force JSON & CBOR to be constructed from io.ReadSeeker
- Ensure only one NodeReader is active at a single time

NodeReader: test Skip and Abort for all readers (and fix it)

coding/Makefile

@@ -4,25 +4,32 @@ bin/multicodec:
 	go get -d github.com/jbenet/go-multicodec/multicodec
 	go build -o "$@" github.com/jbenet/go-multicodec/multicodec
 
+bin/json2cbor:
+	mkdir -p bin
+	go get -d github.com/whyrusleeping/cbor/go/json2cbor
+	go build -o "$@" github.com/whyrusleeping/cbor/go/json2cbor
+
 bin/convert:
+	mkdir -p bin
 	cd bin; go build convert.go
 
 bin/msgio:
 	mkdir -p bin
 	go get -d github.com/jbenet/go-msgio/msgio
 	go build -o "$@" github.com/jbenet/go-msgio/msgio
 
-json.testfile: bin/multicodec Makefile
+json.testfile: bin/multicodec Makefile test1.json
 	: >$@
-	bin/multicodec header /multicodec >>$@
+	bin/multicodec header /mdagv1 >>$@
 	bin/multicodec header /json >>$@
-	echo '{"@codec":"/json","abc":{"mlink":"QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V"}}' >>$@
+	cat test1.json >>$@
 
-cbor.testfile: bin/multicodec json.testfile
-	bin/multicodec header /multicodec >$@
+cbor.testfile: bin/multicodec bin/json2cbor Makefile test1.json
 	./convert -i $< -o $@.tmp -c '/cbor'
-	cat $@.tmp >>$@
-	rm -f $@.tmp
+	: >$@
+	bin/multicodec header /mdagv1 >>$@
+	bin/multicodec header /cbor >>$@
+	bin/json2cbor -i test1.json -o - >>$@
 
 protobuf.testfile: bin/multicodec bin/msgio
 	bin/multicodec header /mdagv1 >$@

coding/base/base_reader.go

@@ -0,0 +1,55 @@
+package base
+
+import (
+	"github.com/ipfs/go-ipld/coding/stream"
+)
+
+type BaseReader struct {
+	Callback  stream.ReadFun
+	cbEnabled []bool
+	path      []interface{}
+	tokens    []stream.ReaderToken
+}
+
+func CreateBaseReader(cb stream.ReadFun) BaseReader {
+	return BaseReader{cb, []bool{}, []interface{}{}, []stream.ReaderToken{}}
+}
+
+// Executes the callback and stay in scope for sub elements
+func (p *BaseReader) ExecCallback(token stream.ReaderToken, value interface{}) error {
+	var err error
+	enabled := !p.Skipping()
+	if enabled {
+		err = p.Callback(p.path, token, value)
+		if err == stream.NodeReadSkip {
+			enabled = false
+			err = nil
+		}
+	}
+	p.cbEnabled = append(p.cbEnabled, enabled)
+	return err
+}
+
+// Return true if a parent callback wants to skip processing of its children
+func (p *BaseReader) Skipping() bool {
+	enabled := true
+	if len(p.cbEnabled) > 0 {
+		enabled = p.cbEnabled[len(p.cbEnabled)-1]
+	}
+	return !enabled
+}
+
+// Must be called after all sub elements below a ExecCallback are processed
+func (p *BaseReader) Descope() {
+	p.cbEnabled = p.cbEnabled[:len(p.cbEnabled)-1]
+}
+
+// Push a path element
+func (p *BaseReader) PushPath(elem interface{}) {
+	p.path = append(p.path, elem)
+}
+
+// Pop a path element
+func (p *BaseReader) PopPath() {
+	p.path = p.path[:len(p.path)-1]
+}

coding/cbor.testfile

@@ -1,3 +1,3 @@
-/multicodec
+/mdagv1
 /cbor
-�cabc�emlinkx.QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V
+�cabc�emlinkx.QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39Vf@codece/json

coding/cbor/encode.go

@@ -0,0 +1,84 @@
+package cbor
+
+import (
+	"io"
+
+	ipld "github.com/ipfs/go-ipld"
+	ma "github.com/jbenet/go-multiaddr"
+	cbor "github.com/whyrusleeping/cbor/go"
+)
+
+// Encode to CBOR, add the multicodec header
+func Encode(w io.Writer, node ipld.NodeIterator, tags bool) error {
+	_, err := w.Write(Header)
+	if err != nil {
+		return err
+	}
+
+	return RawEncode(w, node, tags)
+}
+
+func encodeFilter(val interface{}) interface{} {
+	var ok bool
+	var object map[string]interface{}
+	var link interface{}
+	var linkStr string
+	var newObject map[string]interface{} = map[string]interface{}{}
+	var linkPath interface{}
+	var linkObject interface{}
+
+	object, ok = val.(map[string]interface{})
+	if !ok {
+		return val
+	}
+
+	link, ok = object[ipld.LinkKey]
+	if !ok {
+		return val
+	}
+
+	linkStr, ok = link.(string)
+	if !ok {
+		return val
+	}
+
+	maddr, err := ma.NewMultiaddr(linkStr)
+	for k, v := range object {
+		if k != ipld.LinkKey {
+			newObject[k] = v
+		}
+	}
+	if err != nil || maddr.String() != linkStr {
+		linkPath = linkStr
+	} else {
+		linkPath = maddr.Bytes()
+	}
+
+	if len(newObject) > 0 {
+		linkObject = []interface{}{linkPath, newObject}
+	} else {
+		linkObject = linkPath
+	}
+
+	return &cbor.CBORTag{
+		Tag:           TagIPLDLink,
+		WrappedObject: linkObject,
+	}
+}
+
+// Encode to CBOR, do not add the multicodec header
+func RawEncode(w io.Writer, node ipld.NodeIterator, tags bool) error {
+	enc := cbor.NewEncoder(w)
+	if tags {
+		enc.SetFilter(encodeFilter)
+	}
+
+	// Buffering to memory is absolutely necessary to normalize the CBOR data file
+	// (pairs order in a map)
+	mem, err := ipld.ToMemory(node)
+	if err != nil {
+		return err
+	}
+
+	return enc.Encode(mem)
+}

coding/cbor/links_test.go

@@ -0,0 +1,150 @@
+package cbor
+
+import (
+	"bytes"
+	"testing"
+
+	ipld "github.com/ipfs/go-ipld"
+	reader "github.com/ipfs/go-ipld/coding/stream"
+	readertest "github.com/ipfs/go-ipld/coding/stream/test"
+	memory "github.com/ipfs/go-ipld/memory"
+	multiaddr "github.com/jbenet/go-multiaddr"
+	cbor "github.com/whyrusleeping/cbor/go"
+)
+
+func TestLinksStringEmptyMeta(t *testing.T) {
+	var buf bytes.Buffer
+
+	node, err := reader.NewNodeFromReader(memory.Node{
+		ipld.LinkKey: "#/foo/bar",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = RawEncode(&buf, node.(ipld.NodeIterator), true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	var expected []byte
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeTag, TagIPLDLink, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeText, uint64(len("#/foo/bar")), nil)...)
+	expected = append(expected, []byte("#/foo/bar")...)
+
+	if !bytes.Equal(expected, buf.Bytes()) {
+		t.Error("Incorrect encoding")
+		t.Logf("Expected: %v", expected)
+		t.Logf("Actual:   %v", buf.Bytes())
+	}
+
+	cbor, err := NewCBORDecoder(bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		t.Error(err)
+	}
+
+	readertest.CheckReader(t, cbor, []readertest.Callback{
+		readertest.Cb(readertest.Path(), reader.TokenNode, nil),
+		readertest.Cb(readertest.Path(), reader.TokenKey, ipld.LinkKey),
+		readertest.Cb(readertest.Path(ipld.LinkKey), reader.TokenValue, "#/foo/bar"),
+		readertest.Cb(readertest.Path(), reader.TokenEndNode, nil),
+	})
+}
+
+func TestLinksStringNonEmptyMetaCheckOrdering(t *testing.T) {
+	var buf bytes.Buffer
+
+	node, err := reader.NewNodeFromReader(memory.Node{
+		"size":       55,
+		"00":         11,          // should be encoded first in the map (0 is before s and @ and is smaller)
+		ipld.LinkKey: "#/foo/bar", // should come first, this is a link
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = RawEncode(&buf, node.(ipld.NodeIterator), true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	var expected []byte
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeTag, TagIPLDLink, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeArray, 2, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeText, uint64(len("#/foo/bar")), nil)...)
+	expected = append(expected, []byte("#/foo/bar")...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeMap, 2, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeText, uint64(len("00")), nil)...)
+	expected = append(expected, []byte("00")...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeUint, 11, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeText, uint64(len("size")), nil)...)
+	expected = append(expected, []byte("size")...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeUint, 55, nil)...)
+
+	if !bytes.Equal(expected, buf.Bytes()) {
+		t.Error("Incorrect encoding")
+		t.Logf("Expected: %v", expected)
+		t.Logf("Actual:   %v", buf.Bytes())
+	}
+
+	cbor, err := NewCBORDecoder(bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		t.Error(err)
+	}
+
+	readertest.CheckReader(t, cbor, []readertest.Callback{
+		readertest.Cb(readertest.Path(), reader.TokenNode, nil),
+		readertest.Cb(readertest.Path(), reader.TokenKey, ipld.LinkKey),
+		readertest.Cb(readertest.Path(ipld.LinkKey), reader.TokenValue, "#/foo/bar"),
+		readertest.Cb(readertest.Path(), reader.TokenKey, "00"),
+		readertest.Cb(readertest.Path("00"), reader.TokenValue, uint64(11)),
+		readertest.Cb(readertest.Path(), reader.TokenKey, "size"),
+		readertest.Cb(readertest.Path("size"), reader.TokenValue, uint64(55)),
+		readertest.Cb(readertest.Path(), reader.TokenEndNode, nil),
+	})
+}
+
+func TestLinksMultiAddr(t *testing.T) {
+	var buf bytes.Buffer
+
+	ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/udp/1234")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	node, err := reader.NewNodeFromReader(memory.Node{
+		ipld.LinkKey: ma.String(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = RawEncode(&buf, node.(ipld.NodeIterator), true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	var expected []byte
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeTag, TagIPLDLink, nil)...)
+	expected = append(expected, cbor.EncodeInt(cbor.MajorTypeBytes, uint64(len(ma.Bytes())), nil)...)
+	expected = append(expected, ma.Bytes()...)
+
+	if !bytes.Equal(expected, buf.Bytes()) {
+		t.Error("Incorrect encoding")
+		t.Logf("Expected: %v", expected)
+		t.Logf("Actual:   %v", buf.Bytes())
+	}
+
+	cbor, err := NewCBORDecoder(bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		t.Error(err)
+	}
+
+	readertest.CheckReader(t, cbor, []readertest.Callback{
+		readertest.Cb(readertest.Path(), reader.TokenNode, nil),
+		readertest.Cb(readertest.Path(), reader.TokenKey, ipld.LinkKey),
+		readertest.Cb(readertest.Path(ipld.LinkKey), reader.TokenValue, ma.String()),
+		readertest.Cb(readertest.Path(), reader.TokenEndNode, nil),
+	})
+}