From 844528ceef7bc026b9b9bac0c97d789193854e22 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Tue, 20 Sep 2022 14:17:13 +0200
Subject: [PATCH 01/18] feat(wip): share.WriteEDS and basic tests

---
 go.mod                  |   1 +
 go.sum                  |   2 +
 share/eds.go            | 137 ++++++++++++++++++++++++++++++++++++++++
 share/eds_test.go       |  32 ++++++++++
 share/ipld/nmt_adder.go |  41 ++++++++++--
 5 files changed, 206 insertions(+), 7 deletions(-)
 create mode 100644 share/eds.go
 create mode 100644 share/eds_test.go

diff --git a/go.mod b/go.mod
index 6800ab44e1..8ac59d7494 100644
--- a/go.mod
+++ b/go.mod
@@ -33,6 +33,7 @@ require (
 	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-merkledag v0.7.0
+	github.com/ipld/go-car v0.5.0
 	github.com/libp2p/go-libp2p v0.21.0
 	github.com/libp2p/go-libp2p-core v0.19.1
 	github.com/libp2p/go-libp2p-kad-dht v0.17.0
diff --git a/go.sum b/go.sum
index 65760cbab8..9803632094 100644
--- a/go.sum
+++ b/go.sum
@@ -831,6 +831,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0=
 github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
+github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8=
+github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE=
 github.com/ipld/go-codec-dagpb v1.3.1 h1:yVNlWRQexCa54ln3MSIiUN++ItH7pdhBFhh0hSgZu1w=
 github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY=
 github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
diff --git a/share/eds.go b/share/eds.go
new file mode 100644
index 0000000000..06882cef4a
--- /dev/null
+++ b/share/eds.go
@@ -0,0 +1,137 @@
+package share
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipld/go-car"
+	"github.com/ipld/go-car/util"
+	"github.com/tendermint/tendermint/pkg/wrapper"
+
+	"github.com/celestiaorg/celestia-node/ipld"
+	"github.com/celestiaorg/celestia-node/ipld/plugin"
+
+	"github.com/celestiaorg/nmt"
+	"github.com/celestiaorg/rsmt2d"
+)
+
+// WriteEDS writes the whole EDS into the given io.Writer as CARv1 file.
+// All its shares and recomputed NMT proofs.
+func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
+	// 1. Reimport EDS. This is needed to get the proofs.
+	//    - Using Blockservice w/ offline exchange and in-memory blockstore.
+	//    - With NodeVisitor, which saves ONLY PROOFS to the blockstore
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	store := blockstore.NewBlockstore(dstore)
+	bs := blockservice.New(store, nil)
+	shares := ipld.ExtractEDS(eds)
+	if len(shares) == 0 {
+		return fmt.Errorf("ipld: importing empty data")
+	}
+	squareSize := int(math.Sqrt(float64(len(shares))))
+	// todo: batch size here
+	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize))
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
+	eds, err := rsmt2d.ImportExtendedDataSquare(shares, ipld.DefaultRSMT2DCodec(), tree.Constructor)
+	if err != nil {
+		return fmt.Errorf("failure to recompute the extended data square: %w", err)
+	}
+	// compute roots
+	eds.RowRoots()
+	eds.ColRoots()
+	// commit the batch to DAG
+	err = batchAdder.Commit()
+	if err != nil {
+		return fmt.Errorf("failure to commit the inner nodes to the dag: %w", err)
+	}
+
+	// 2. Creates and writes Carv1Header
+	//    - Roots are the eds Row + Col roots
+	rootCids, err := rootsToCids(eds)
+	if err != nil {
+		return fmt.Errorf("failure to get root cids: %w", err)
+	}
+	err = car.WriteHeader(&car.CarHeader{
+		Roots:   rootCids,
+		Version: 1,
+	}, w)
+	if err != nil {
+		return fmt.Errorf("failure to write carv1 header: %w", err)
+	}
+
+	// 3. Iterates over shares in quadrant order vis eds.GetCell
+	//    - Writes the shares in row-by-row order
+
+	// TODO: Write the shares in quadrant order. PROBLEM: Need the CIDs from the adder
+	//shares, err = quadrantOrder(eds)
+	//if err != nil {
+	//	return fmt.Errorf("failure to get shares in quadrant order: %w", err)
+	//}
+	leaves := batchAdder.Leaves()
+	leafMap := batchAdder.LeafMap()
+	for _, leafCid := range leaves.Keys() {
+		if err != nil {
+			return fmt.Errorf("failure to get cid from share: %w", err)
+		}
+		err = util.LdWrite(w, leafCid.Bytes(), leafMap[leafCid])
+		if err != nil {
+			return fmt.Errorf("failure to write share: %w", err)
+		}
+	}
+
+	// 4. Iterates over in-memory Blockstore and writes proofs to the CAR
+	proofs, err := store.AllKeysChan(ctx)
+	if err != nil {
+		return fmt.Errorf("failure to get all keys from the blockstore: %w", err)
+	}
+	for proofCid := range proofs {
+		node, err := store.Get(ctx, proofCid)
+		if err != nil {
+			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
+		}
+		err = util.LdWrite(w, proofCid.Bytes(), node.RawData())
+		if err != nil {
+			return fmt.Errorf("failure to write proof to the car: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) ([][]byte, error) {
+	size := eds.Width() * eds.Width()
+	shares := make([][]byte, size)
+
+	rowCount := eds.Width() / 2
+	// TODO: Simplify this loop
+	for i := 0; i < int(rowCount); i++ {
+		for j := 0; j < int(rowCount); j++ {
+			shares[(0*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i), uint(j))
+			shares[(1*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i), uint(j)+rowCount)
+			shares[(2*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i)+rowCount, uint(j))
+			shares[(3*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i)+rowCount, uint(j)+rowCount)
+		}
+	}
+	return shares, nil
+}
+
+func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
+	var err error
+	roots := append(eds.RowRoots(), eds.ColRoots()...)
+	rootCids := make([]cid.Cid, len(roots))
+	for i, r := range roots {
+		rootCids[i], err = plugin.CidFromNamespacedSha256(r)
+		if err != nil {
+			return nil, fmt.Errorf("failure to get cid from root: %w", err)
+		}
+	}
+	return rootCids, nil
+}
diff --git a/share/eds_test.go b/share/eds_test.go
new file mode 100644
index 0000000000..946dbdbb1e
--- /dev/null
+++ b/share/eds_test.go
@@ -0,0 +1,32 @@
+package share
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/celestiaorg/celestia-node/ipld"
+)
+
+func TestQuadrantOrder(t *testing.T) {
+	//result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{
+	//	{1}, {2},
+	//	{3}, {4},
+	//}, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
+	// TODO: make this into an actual test
+	//fmt.Println(quadrantOrder(result))
+}
+
+func TestWriteEDS(t *testing.T) {
+	f, err := os.OpenFile("/tmp/123.car", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	eds := ipld.RandEDS(t, 4)
+	err = WriteEDS(context.Background(), eds, f)
+	require.Nil(t, err)
+}
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 0925a8094c..0c4335a09a 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -12,10 +12,11 @@ import (
 // NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted
 // into a nmt tree.
 type NmtNodeAdder struct {
-	ctx    context.Context
-	add    *ipld.Batch
-	leaves *cid.Set
-	err    error
+	ctx     context.Context
+	add     *ipld.Batch
+	leaves  *cid.Set
+	leafMap map[cid.Cid][]byte
+	err     error
 }
 
 // NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and
@@ -23,9 +24,10 @@ type NmtNodeAdder struct {
 // It is not thread-safe.
 func NewNmtNodeAdder(ctx context.Context, bs blockservice.BlockService, opts ...ipld.BatchOption) *NmtNodeAdder {
 	return &NmtNodeAdder{
-		add:    ipld.NewBatch(ctx, merkledag.NewDAGService(bs), opts...),
-		ctx:    ctx,
-		leaves: cid.NewSet(),
+		add:     ipld.NewBatch(ctx, merkledag.NewDAGService(bs), opts...),
+		ctx:     ctx,
+		leafMap: make(map[cid.Cid][]byte),
+		leaves:  cid.NewSet(),
 	}
 }
 
@@ -48,6 +50,31 @@ func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) {
 	}
 }
 
+func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
+	if n.err != nil {
+		return // protect from further visits if there is an error
+	}
+
+	id := plugin.MustCidFromNamespacedSha256(hash)
+	switch len(children) {
+	case 1:
+		n.leaves.Add(id)
+		n.leafMap[id] = children[0]
+	case 2:
+		n.err = n.add.Add(n.ctx, plugin.NewNMTNode(id, children[0], children[1]))
+	default:
+		panic("expected a binary tree")
+	}
+}
+
+func (n *NmtNodeAdder) LeafMap() map[cid.Cid][]byte {
+	return n.leafMap
+}
+
+func (n *NmtNodeAdder) Leaves() *cid.Set {
+	return n.leaves
+}
+
 // Commit checks for errors happened during Visit and if absent commits data to inner Batch.
 func (n *NmtNodeAdder) Commit() error {
 	if n.err != nil {

From 4ce9f6c06e0b821d66906075a879f2b982575116 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Wed, 21 Sep 2022 14:36:41 +0200
Subject: [PATCH 02/18] refactor(blocksync): write shares in quadrant order

---
 share/eds.go            | 18 ++++++++----------
 share/eds_test.go       |  2 +-
 share/ipld/nmt_adder.go | 27 ++++++++-------------------
 3 files changed, 17 insertions(+), 30 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 06882cef4a..855a0c9dcf 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -69,19 +69,17 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 
 	// 3. Iterates over shares in quadrant order vis eds.GetCell
 	//    - Writes the shares in row-by-row order
-
-	// TODO: Write the shares in quadrant order. PROBLEM: Need the CIDs from the adder
-	//shares, err = quadrantOrder(eds)
-	//if err != nil {
-	//	return fmt.Errorf("failure to get shares in quadrant order: %w", err)
-	//}
-	leaves := batchAdder.Leaves()
-	leafMap := batchAdder.LeafMap()
-	for _, leafCid := range leaves.Keys() {
+	shares, err = quadrantOrder(eds)
+	fmt.Println(shares)
+	if err != nil {
+		return fmt.Errorf("failure to get shares in quadrant order: %w", err)
+	}
+	for _, share := range shares {
+		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
 		}
-		err = util.LdWrite(w, leafCid.Bytes(), leafMap[leafCid])
+		err = util.LdWrite(w, cid.Bytes(), share)
 		if err != nil {
 			return fmt.Errorf("failure to write share: %w", err)
 		}
diff --git a/share/eds_test.go b/share/eds_test.go
index 946dbdbb1e..35326b81f8 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -26,7 +26,7 @@ func TestWriteEDS(t *testing.T) {
 	}
 	defer f.Close()
 
-	eds := ipld.RandEDS(t, 4)
+	eds := ipld.RandEDS(t, 2)
 	err = WriteEDS(context.Background(), eds, f)
 	require.Nil(t, err)
 }
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 0c4335a09a..44c759a955 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -12,11 +12,10 @@ import (
 // NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted
 // into a nmt tree.
 type NmtNodeAdder struct {
-	ctx     context.Context
-	add     *ipld.Batch
-	leaves  *cid.Set
-	leafMap map[cid.Cid][]byte
-	err     error
+	ctx    context.Context
+	add    *ipld.Batch
+	leaves *cid.Set
+	err    error
 }
 
 // NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and
@@ -24,10 +23,9 @@ type NmtNodeAdder struct {
 // It is not thread-safe.
 func NewNmtNodeAdder(ctx context.Context, bs blockservice.BlockService, opts ...ipld.BatchOption) *NmtNodeAdder {
 	return &NmtNodeAdder{
-		add:     ipld.NewBatch(ctx, merkledag.NewDAGService(bs), opts...),
-		ctx:     ctx,
-		leafMap: make(map[cid.Cid][]byte),
-		leaves:  cid.NewSet(),
+		add:    ipld.NewBatch(ctx, merkledag.NewDAGService(bs), opts...),
+		ctx:    ctx,
+		leaves: cid.NewSet(),
 	}
 }
 
@@ -58,8 +56,7 @@ func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
 	id := plugin.MustCidFromNamespacedSha256(hash)
 	switch len(children) {
 	case 1:
-		n.leaves.Add(id)
-		n.leafMap[id] = children[0]
+		break
 	case 2:
 		n.err = n.add.Add(n.ctx, plugin.NewNMTNode(id, children[0], children[1]))
 	default:
@@ -67,14 +64,6 @@ func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
 	}
 }
 
-func (n *NmtNodeAdder) LeafMap() map[cid.Cid][]byte {
-	return n.leafMap
-}
-
-func (n *NmtNodeAdder) Leaves() *cid.Set {
-	return n.leaves
-}
-
 // Commit checks for errors happened during Visit and if absent commits data to inner Batch.
 func (n *NmtNodeAdder) Commit() error {
 	if n.err != nil {

From dd7568fc7a188af1732089f8d0c7f0c3f24f18e0 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Wed, 21 Sep 2022 14:53:46 +0200
Subject: [PATCH 03/18] fix(blocksync): fixing CIDs and quadrantOrder algorithm

---
 share/eds.go | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 855a0c9dcf..99bad03d0b 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -46,7 +46,6 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	}
 	// compute roots
 	eds.RowRoots()
-	eds.ColRoots()
 	// commit the batch to DAG
 	err = batchAdder.Commit()
 	if err != nil {
@@ -68,12 +67,7 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	}
 
 	// 3. Iterates over shares in quadrant order vis eds.GetCell
-	//    - Writes the shares in row-by-row order
-	shares, err = quadrantOrder(eds)
-	fmt.Println(shares)
-	if err != nil {
-		return fmt.Errorf("failure to get shares in quadrant order: %w", err)
-	}
+	shares = quadrantOrder(eds)
 	for _, share := range shares {
 		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
 		if err != nil {
@@ -95,7 +89,11 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 		if err != nil {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
-		err = util.LdWrite(w, proofCid.Bytes(), node.RawData())
+		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()))
+		if err != nil {
+			return fmt.Errorf("failure to get cid: %w", err)
+		}
+		err = util.LdWrite(w, cid.Bytes(), node.RawData())
 		if err != nil {
 			return fmt.Errorf("failure to write proof to the car: %w", err)
 		}
@@ -104,21 +102,22 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	return nil
 }
 
-func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) ([][]byte, error) {
+func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 	size := eds.Width() * eds.Width()
 	shares := make([][]byte, size)
 
-	rowCount := eds.Width() / 2
+	quadrantWidth := int(eds.Width() / 2)
+	quadrantSize := quadrantWidth * quadrantWidth
 	// TODO: Simplify this loop
-	for i := 0; i < int(rowCount); i++ {
-		for j := 0; j < int(rowCount); j++ {
-			shares[(0*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i), uint(j))
-			shares[(1*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i), uint(j)+rowCount)
-			shares[(2*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i)+rowCount, uint(j))
-			shares[(3*int(eds.Width()))+i*int(rowCount)+j] = eds.GetCell(uint(i)+rowCount, uint(j)+rowCount)
+	for i := 0; i < quadrantWidth; i++ {
+		for j := 0; j < quadrantWidth; j++ {
+			shares[(0*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i), uint(j))
+			shares[(1*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i), uint(j+quadrantWidth))
+			shares[(2*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i+quadrantWidth), uint(j))
+			shares[(3*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))
 		}
 	}
-	return shares, nil
+	return shares
 }
 
 func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {

From 2ab2fbf9706de7febced2a4d5903e2d626389448 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Thu, 22 Sep 2022 09:24:47 +0200
Subject: [PATCH 04/18] fix(blocksync): retrieving inner node CIDs correctly

---
 share/eds.go            | 5 +++--
 share/eds_test.go       | 3 +--
 share/ipld/nmt_adder.go | 1 -
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 99bad03d0b..2a69b8a70d 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -38,7 +38,7 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	}
 	squareSize := int(math.Sqrt(float64(len(shares))))
 	// todo: batch size here
-	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize))
+	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize/2))
 	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
 	eds, err := rsmt2d.ImportExtendedDataSquare(shares, ipld.DefaultRSMT2DCodec(), tree.Constructor)
 	if err != nil {
@@ -89,7 +89,8 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 		if err != nil {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
-		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()))
+		// TODO: Learn why this doesn't match proofCid or node.Cid()
+		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()[1:]))
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)
 		}
diff --git a/share/eds_test.go b/share/eds_test.go
index 35326b81f8..2531875e66 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -2,12 +2,11 @@ package share
 
 import (
 	"context"
+	"github.com/celestiaorg/celestia-node/ipld"
 	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-
-	"github.com/celestiaorg/celestia-node/ipld"
 )
 
 func TestQuadrantOrder(t *testing.T) {
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 44c759a955..6649e8169a 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -2,7 +2,6 @@ package ipld
 
 import (
 	"context"
-
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"

From 8135709488cd707484a770043bab6bd3082d0ff4 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Thu, 22 Sep 2022 10:07:30 +0200
Subject: [PATCH 05/18] refactor(blocksync): split up WriteEDS into
 subroutines, introduction of writingSession

---
 share/eds.go            | 103 ++++++++++++++++++++++++++++++----------
 share/eds_test.go       |  13 ++---
 share/ipld/nmt_adder.go |   1 +
 3 files changed, 86 insertions(+), 31 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 2a69b8a70d..bdb7c4f47b 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -23,69 +23,122 @@ import (
 	"github.com/celestiaorg/rsmt2d"
 )
 
-// WriteEDS writes the whole EDS into the given io.Writer as CARv1 file.
-// All its shares and recomputed NMT proofs.
+// writingSession contains the components needed to write an EDS to a CARv1 file with our custom node order.
+type writingSession struct {
+	eds *rsmt2d.ExtendedDataSquare
+	// store is an in-memory blockstore, used to cache the inner nodes (proofs) while we walk the nmt tree.
+	store blockstore.Blockstore
+	w     io.Writer
+}
+
+// WriteEDS writes the entire EDS into the given io.Writer as CARv1 file.
+// This includes all shares in quadrant order, followed by all inner nodes of the NMT tree.
+// Order: Carv1Header - Q1 - Q2 - Q3 - Q4 - InnerNodes
 func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
-	// 1. Reimport EDS. This is needed to get the proofs.
-	//    - Using Blockservice w/ offline exchange and in-memory blockstore.
-	//    - With NodeVisitor, which saves ONLY PROOFS to the blockstore
-	dstore := dssync.MutexWrap(ds.NewMapDatastore())
-	store := blockstore.NewBlockstore(dstore)
+	// 1. Reimport EDS. This is needed to traverse the NMT tree and cache the inner nodes (proofs)
+	writer, err := initializeWriter(ctx, eds, w)
+	if err != nil {
+		return fmt.Errorf("failure creating eds writer: %w", err)
+	}
+
+	// 2. Creates and writes Carv1Header
+	//    - Roots are the eds Row + Col roots
+	err = writer.writeHeader()
+	if err != nil {
+		return fmt.Errorf("failure writing carv1 header: %w", err)
+	}
+
+	// 3. Iterates over shares in quadrant order vis eds.GetCell
+	err = writer.writeShares()
+	if err != nil {
+		return fmt.Errorf("failure writing shares: %w", err)
+	}
+
+	// 4. Iterates over in-memory Blockstore and writes proofs to the CAR
+	err = writer.writeProofs(ctx)
+	if err != nil {
+		return fmt.Errorf("failure writing proofs: %w", err)
+	}
+
+	return nil
+}
+
+// initializeWriter reimports the EDS into an in-memory blockstore in order to cache the proofs.
+func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) (*writingSession, error) {
+	// we use an in-memory blockstore and an offline exchange
+	store := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
 	bs := blockservice.New(store, nil)
+	// shares are extracted from the eds so that we can reimport them to traverse
 	shares := ipld.ExtractEDS(eds)
 	if len(shares) == 0 {
-		return fmt.Errorf("ipld: importing empty data")
+		return nil, fmt.Errorf("ipld: importing empty data")
 	}
+	// todo: add correct batch size here
 	squareSize := int(math.Sqrt(float64(len(shares))))
-	// todo: batch size here
 	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize/2))
+	// this adder ignores leaves, so that they are not added to the store we iterate through in writeProofs
 	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
 	eds, err := rsmt2d.ImportExtendedDataSquare(shares, ipld.DefaultRSMT2DCodec(), tree.Constructor)
 	if err != nil {
-		return fmt.Errorf("failure to recompute the extended data square: %w", err)
+		return nil, fmt.Errorf("failure to recompute the extended data square: %w", err)
 	}
 	// compute roots
 	eds.RowRoots()
 	// commit the batch to DAG
 	err = batchAdder.Commit()
 	if err != nil {
-		return fmt.Errorf("failure to commit the inner nodes to the dag: %w", err)
+		return nil, fmt.Errorf("failure to commit the inner nodes to the dag: %w", err)
 	}
 
-	// 2. Creates and writes Carv1Header
-	//    - Roots are the eds Row + Col roots
-	rootCids, err := rootsToCids(eds)
+	return &writingSession{
+		eds:   eds,
+		store: store,
+		w:     w,
+	}, nil
+}
+
+// writeHeader creates a CarV1 header using the EDS's Row and Column roots as the list of DAG roots.
+func (w *writingSession) writeHeader() error {
+	rootCids, err := rootsToCids(w.eds)
 	if err != nil {
-		return fmt.Errorf("failure to get root cids: %w", err)
+		return fmt.Errorf("failure getting root cids: %w", err)
 	}
+
 	err = car.WriteHeader(&car.CarHeader{
 		Roots:   rootCids,
 		Version: 1,
-	}, w)
+	}, w.w)
 	if err != nil {
-		return fmt.Errorf("failure to write carv1 header: %w", err)
+		return fmt.Errorf("failure writing carv1 header: %w", err)
 	}
+	return nil
+}
 
-	// 3. Iterates over shares in quadrant order vis eds.GetCell
-	shares = quadrantOrder(eds)
+// writeShares reorders the shares to quadrant order and writes them to the CARv1 file.
+func (w *writingSession) writeShares() error {
+	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
 		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
 		}
-		err = util.LdWrite(w, cid.Bytes(), share)
+		err = util.LdWrite(w.w, cid.Bytes(), share)
 		if err != nil {
 			return fmt.Errorf("failure to write share: %w", err)
 		}
 	}
+	return nil
+}
 
-	// 4. Iterates over in-memory Blockstore and writes proofs to the CAR
-	proofs, err := store.AllKeysChan(ctx)
+// writeProofs iterates over the in-memory blockstore's keys and writes all inner nodes to the CARv1 file.
+func (w *writingSession) writeProofs(ctx context.Context) error {
+	// we only stored proofs to the store, so we can just iterate over them here without getting any leaves
+	proofs, err := w.store.AllKeysChan(ctx)
 	if err != nil {
 		return fmt.Errorf("failure to get all keys from the blockstore: %w", err)
 	}
 	for proofCid := range proofs {
-		node, err := store.Get(ctx, proofCid)
+		node, err := w.store.Get(ctx, proofCid)
 		if err != nil {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
@@ -94,12 +147,11 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)
 		}
-		err = util.LdWrite(w, cid.Bytes(), node.RawData())
+		err = util.LdWrite(w.w, cid.Bytes(), node.RawData())
 		if err != nil {
 			return fmt.Errorf("failure to write proof to the car: %w", err)
 		}
 	}
-
 	return nil
 }
 
@@ -121,6 +173,7 @@ func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 	return shares
 }
 
+// rootsToCids converts the EDS's Row and Column roots to CIDs.
 func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
 	var err error
 	roots := append(eds.RowRoots(), eds.ColRoots()...)
diff --git a/share/eds_test.go b/share/eds_test.go
index 2531875e66..1bed506b3d 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -19,13 +19,14 @@ func TestQuadrantOrder(t *testing.T) {
 }
 
 func TestWriteEDS(t *testing.T) {
-	f, err := os.OpenFile("/tmp/123.car", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
-	if err != nil {
-		panic(err)
-	}
+	tmpDir := t.TempDir()
+	err := os.Chdir(tmpDir)
+	require.NoError(t, err, "error changing to the temporary test directory")
+	f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+	require.NoError(t, err, "error opening file")
 	defer f.Close()
 
-	eds := ipld.RandEDS(t, 2)
+	eds := ipld.RandEDS(t, 4)
 	err = WriteEDS(context.Background(), eds, f)
-	require.Nil(t, err)
+	require.NoError(t, err, "error writing EDS to file")
 }
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 6649e8169a..0be748e0c0 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -47,6 +47,7 @@ func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) {
 	}
 }
 
+// VisitInnerNodes is a NodeVisitor that does not store leaf nodes to the blockservice.
 func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
 	if n.err != nil {
 		return // protect from further visits if there is an error

From 7451659f58ac1df188002a114b659e72a63b3f9b Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Thu, 22 Sep 2022 14:24:45 +0200
Subject: [PATCH 06/18] fix(blocksync): adding tests, fixing shares according
 to nmt wrapper

---
 share/eds.go      | 16 +++++---
 share/eds_test.go | 98 +++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 101 insertions(+), 13 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index bdb7c4f47b..c1c801b55c 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -3,6 +3,7 @@ package share
 import (
 	"context"
 	"fmt"
+	"github.com/tendermint/tendermint/pkg/consts"
 	"io"
 	"math"
 
@@ -118,7 +119,10 @@ func (w *writingSession) writeHeader() error {
 func (w *writingSession) writeShares() error {
 	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
-		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
+		// TODO: Okay. this is really weird and I don't understand:
+		// We need to cut off the first byte like we do for inner nodes, but this share doesn't even have the prefix...
+		// So what is going on? If we don't do this, the cid doesn't match on read.
+		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share[1:]))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
 		}
@@ -155,6 +159,7 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 	return nil
 }
 
+// quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, adding the wrapped namespace
 func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 	size := eds.Width() * eds.Width()
 	shares := make([][]byte, size)
@@ -164,10 +169,11 @@ func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 	// TODO: Simplify this loop
 	for i := 0; i < quadrantWidth; i++ {
 		for j := 0; j < quadrantWidth; j++ {
-			shares[(0*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i), uint(j))
-			shares[(1*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i), uint(j+quadrantWidth))
-			shares[(2*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i+quadrantWidth), uint(j))
-			shares[(3*quadrantSize)+i*quadrantWidth+j] = eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))
+			q0Cell := eds.GetCell(uint(i), uint(j))
+			shares[(0*quadrantSize)+i*quadrantWidth+j] = append(q0Cell[:8], q0Cell...)
+			shares[(1*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i), uint(j+quadrantWidth))...)
+			shares[(2*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j))...)
+			shares[(3*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))...)
 		}
 	}
 	return shares
diff --git a/share/eds_test.go b/share/eds_test.go
index 1bed506b3d..a6a9e8397d 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -3,6 +3,10 @@ package share
 import (
 	"context"
 	"github.com/celestiaorg/celestia-node/ipld"
+	"github.com/celestiaorg/rsmt2d"
+	ds "github.com/ipfs/go-datastore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	carv1 "github.com/ipld/go-car"
 	"os"
 	"testing"
 
@@ -10,23 +14,101 @@ import (
 )
 
 func TestQuadrantOrder(t *testing.T) {
-	//result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{
-	//	{1}, {2},
-	//	{3}, {4},
-	//}, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
-	// TODO: make this into an actual test
-	//fmt.Println(quadrantOrder(result))
+	// TODO: add more test cases
+	result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{
+		{1}, {2},
+		{3}, {4},
+	}, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
+	//{{1}, {2}, {7}, {13}},
+	//{{3}, {4}, {13}, {31}},
+	//{{5}, {14}, {19}, {41}},
+	//{{9}, {26}, {47}, {69}},
+	require.Equal(t,
+		[][]byte{
+			{1}, {2}, {3}, {4},
+			{7}, {13}, {13}, {31},
+			{5}, {14}, {9}, {26},
+			{19}, {41}, {47}, {69},
+		}, quadrantOrder(result),
+	)
 }
 
 func TestWriteEDS(t *testing.T) {
+	writeRandomEDS(t)
+}
+
+func TestWriteEDSHeaderRoots(t *testing.T) {
+	eds := writeRandomEDS(t)
+	f := openWrittenEDS(t)
+	defer f.Close()
+
+	reader, err := carv1.NewCarReader(f)
+	require.NoError(t, err, "error creating car reader")
+	roots, err := rootsToCids(eds)
+	require.NoError(t, err, "error converting roots to cids")
+	require.Equal(t, roots, reader.Header.Roots)
+}
+
+func TestWriteEDSStartsWithLeaves(t *testing.T) {
+	eds := writeRandomEDS(t)
+	f := openWrittenEDS(t)
+	defer f.Close()
+
+	reader, err := carv1.NewCarReader(f)
+	require.NoError(t, err, "error creating car reader")
+	block, err := reader.Next()
+	require.NoError(t, err, "error getting first block")
+
+	require.Equal(t, block.RawData()[ipld.NamespaceSize:], eds.GetCell(0, 0))
+}
+
+func TestWriteEDSIncludesRoots(t *testing.T) {
+	writeRandomEDS(t)
+	f := openWrittenEDS(t)
+	defer f.Close()
+
+	bs := blockstore.NewBlockstore(ds.NewMapDatastore())
+	loaded, err := carv1.LoadCar(context.Background(), bs, f)
+	require.NoError(t, err, "error loading car file")
+	for _, root := range loaded.Roots {
+		ok, err := bs.Has(context.Background(), root)
+		require.NoError(t, err, "error checking if blockstore has root")
+		require.True(t, ok, "blockstore does not have root")
+	}
+}
+
+func TestWriteEDSInQuadrantOrder(t *testing.T) {
+	eds := writeRandomEDS(t)
+	f := openWrittenEDS(t)
+	defer f.Close()
+
+	reader, err := carv1.NewCarReader(f)
+	require.NoError(t, err, "error creating car reader")
+
+	shares := quadrantOrder(eds)
+	for i := 0; i < len(shares); i++ {
+		block, err := reader.Next()
+		require.NoError(t, err, "error getting block")
+		require.Equal(t, block.RawData(), shares[i])
+	}
+}
+
+func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare {
 	tmpDir := t.TempDir()
 	err := os.Chdir(tmpDir)
 	require.NoError(t, err, "error changing to the temporary test directory")
-	f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+	f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE, 0600)
 	require.NoError(t, err, "error opening file")
-	defer f.Close()
 
 	eds := ipld.RandEDS(t, 4)
 	err = WriteEDS(context.Background(), eds, f)
 	require.NoError(t, err, "error writing EDS to file")
+	f.Close()
+	return eds
+}
+
+func openWrittenEDS(t *testing.T) *os.File {
+	f, err := os.OpenFile("test.car", os.O_RDONLY, 0600)
+	require.NoError(t, err, "error opening file")
+	return f
 }

From 744fd73af9c25c59dd02b3e7d0645316e3d6b571 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Thu, 22 Sep 2022 14:52:05 +0200
Subject: [PATCH 07/18] chore(lint): goimports and line length

---
 share/eds.go            | 12 ++++++++----
 share/eds_test.go       | 18 ++++++++++--------
 share/ipld/nmt_adder.go |  1 +
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index c1c801b55c..712710b6f9 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -3,10 +3,11 @@ package share
 import (
 	"context"
 	"fmt"
-	"github.com/tendermint/tendermint/pkg/consts"
 	"io"
 	"math"
 
+	"github.com/tendermint/tendermint/pkg/consts"
+
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
@@ -171,9 +172,12 @@ func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 		for j := 0; j < quadrantWidth; j++ {
 			q0Cell := eds.GetCell(uint(i), uint(j))
 			shares[(0*quadrantSize)+i*quadrantWidth+j] = append(q0Cell[:8], q0Cell...)
-			shares[(1*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i), uint(j+quadrantWidth))...)
-			shares[(2*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j))...)
-			shares[(3*quadrantSize)+i*quadrantWidth+j] = append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))...)
+			shares[(1*quadrantSize)+i*quadrantWidth+j] =
+				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i), uint(j+quadrantWidth))...)
+			shares[(2*quadrantSize)+i*quadrantWidth+j] =
+				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j))...)
+			shares[(3*quadrantSize)+i*quadrantWidth+j] =
+				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))...)
 		}
 	}
 	return shares
diff --git a/share/eds_test.go b/share/eds_test.go
index a6a9e8397d..27923db742 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -2,13 +2,15 @@ package share
 
 import (
 	"context"
-	"github.com/celestiaorg/celestia-node/ipld"
-	"github.com/celestiaorg/rsmt2d"
+	"os"
+	"testing"
+
 	ds "github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carv1 "github.com/ipld/go-car"
-	"os"
-	"testing"
+
+	"github.com/celestiaorg/celestia-node/ipld"
+	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/stretchr/testify/require"
 )
@@ -19,10 +21,10 @@ func TestQuadrantOrder(t *testing.T) {
 		{1}, {2},
 		{3}, {4},
 	}, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
-	//{{1}, {2}, {7}, {13}},
-	//{{3}, {4}, {13}, {31}},
-	//{{5}, {14}, {19}, {41}},
-	//{{9}, {26}, {47}, {69}},
+	//  {{1}, {2}, {7}, {13}},
+	//  {{3}, {4}, {13}, {31}},
+	//  {{5}, {14}, {19}, {41}},
+	//  {{9}, {26}, {47}, {69}},
 	require.Equal(t,
 		[][]byte{
 			{1}, {2}, {3}, {4},
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 0be748e0c0..24b78c14ae 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -2,6 +2,7 @@ package ipld
 
 import (
 	"context"
+
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"

From 1cf2873793556d821f1e1e4a653316771326ccc2 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Thu, 22 Sep 2022 16:03:29 +0200
Subject: [PATCH 08/18] fix(blocksync): fixing TestQuadrantOrder to include
 namespace

---
 share/eds_test.go | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/share/eds_test.go b/share/eds_test.go
index 27923db742..50ff313939 100644
--- a/share/eds_test.go
+++ b/share/eds_test.go
@@ -8,6 +8,7 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carv1 "github.com/ipld/go-car"
+	"github.com/tendermint/tendermint/pkg/consts"
 
 	"github.com/celestiaorg/celestia-node/ipld"
 	"github.com/celestiaorg/rsmt2d"
@@ -17,9 +18,12 @@ import (
 
 func TestQuadrantOrder(t *testing.T) {
 	// TODO: add more test cases
+	nID := []byte{0, 0, 0, 0, 0, 0, 0, 0}
+	parity := append(consts.ParitySharesNamespaceID, nID...) //nolint
+	doubleNID := append(nID, nID...)                         //nolint
 	result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{
-		{1}, {2},
-		{3}, {4},
+		append(nID, 1), append(nID, 2),
+		append(nID, 3), append(nID, 4),
 	}, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
 	//  {{1}, {2}, {7}, {13}},
 	//  {{3}, {4}, {13}, {31}},
@@ -27,10 +31,10 @@ func TestQuadrantOrder(t *testing.T) {
 	//  {{9}, {26}, {47}, {69}},
 	require.Equal(t,
 		[][]byte{
-			{1}, {2}, {3}, {4},
-			{7}, {13}, {13}, {31},
-			{5}, {14}, {9}, {26},
-			{19}, {41}, {47}, {69},
+			append(doubleNID, 1), append(doubleNID, 2), append(doubleNID, 3), append(doubleNID, 4),
+			append(parity, 7), append(parity, 13), append(parity, 13), append(parity, 31),
+			append(parity, 5), append(parity, 14), append(parity, 9), append(parity, 26),
+			append(parity, 19), append(parity, 41), append(parity, 47), append(parity, 69),
 		}, quadrantOrder(result),
 	)
 }

From 1bb6be42a01a767bafbadeeb752e1aaf5176cf2a Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Mon, 26 Sep 2022 16:14:42 +0200
Subject: [PATCH 09/18] refactor(share): making quadrantOrder loop more
 readable

---
 share/eds.go | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 712710b6f9..5622508dde 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -167,22 +167,42 @@ func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 
 	quadrantWidth := int(eds.Width() / 2)
 	quadrantSize := quadrantWidth * quadrantWidth
-	// TODO: Simplify this loop
 	for i := 0; i < quadrantWidth; i++ {
 		for j := 0; j < quadrantWidth; j++ {
-			q0Cell := eds.GetCell(uint(i), uint(j))
-			shares[(0*quadrantSize)+i*quadrantWidth+j] = append(q0Cell[:8], q0Cell...)
-			shares[(1*quadrantSize)+i*quadrantWidth+j] =
-				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i), uint(j+quadrantWidth))...)
-			shares[(2*quadrantSize)+i*quadrantWidth+j] =
-				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j))...)
-			shares[(3*quadrantSize)+i*quadrantWidth+j] =
-				append(consts.ParitySharesNamespaceID, eds.GetCell(uint(i+quadrantWidth), uint(j+quadrantWidth))...)
+			cells := getQuadrantCells(eds, uint(i), uint(j))
+			innerOffset := i*quadrantWidth + j
+			for quadrant := 0; quadrant < 4; quadrant++ {
+				shares[(quadrant*quadrantSize)+innerOffset] = prependNamespace(quadrant, cells[quadrant])
+			}
 		}
 	}
 	return shares
 }
 
+// getQuadrantCells returns the cell of each EDS quadrant with the passed inner-quadrant coordinates
+func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte {
+	cells := make([][]byte, 4)
+	quadrantWidth := eds.Width() / 2
+	cells[0] = eds.GetCell(i, j)
+	cells[1] = eds.GetCell(i, j+quadrantWidth)
+	cells[2] = eds.GetCell(i+quadrantWidth, j)
+	cells[3] = eds.GetCell(i+quadrantWidth, j+quadrantWidth)
+	return cells
+}
+
+// prependNamespace adds the namespace to the passed share if in the first quadrant,
+// otherwise it adds the ParitySharesNamespace to the beginning.
+func prependNamespace(quadrant int, share []byte) []byte {
+	switch quadrant {
+	case 0:
+		return append(share[:8], share...)
+	case 1, 2, 3:
+		return append(consts.ParitySharesNamespaceID, share...)
+	default:
+		panic("invalid quadrant")
+	}
+}
+
 // rootsToCids converts the EDS's Row and Column roots to CIDs.
 func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
 	var err error

From 30ed57c1d31c569938d8a4fea7775bafd5aa5c5c Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Fri, 30 Sep 2022 12:49:01 +0200
Subject: [PATCH 10/18] Apply suggestions from code review

Co-authored-by: Viacheslav <viacheslavgonkivskyi@gmail.com>
---
 share/eds.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 5622508dde..7e9c5dd96c 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -40,26 +40,26 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	// 1. Reimport EDS. This is needed to traverse the NMT tree and cache the inner nodes (proofs)
 	writer, err := initializeWriter(ctx, eds, w)
 	if err != nil {
-		return fmt.Errorf("failure creating eds writer: %w", err)
+                return fmt.Errorf("share: failure creating eds writer: %w", err)
 	}
 
 	// 2. Creates and writes Carv1Header
 	//    - Roots are the eds Row + Col roots
 	err = writer.writeHeader()
 	if err != nil {
-		return fmt.Errorf("failure writing carv1 header: %w", err)
+                return fmt.Errorf("share: failure writing carv1 header: %w", err)
 	}
 
 	// 3. Iterates over shares in quadrant order vis eds.GetCell
 	err = writer.writeShares()
 	if err != nil {
-		return fmt.Errorf("failure writing shares: %w", err)
+                return fmt.Errorf("share: failure writing shares: %w", err)
 	}
 
 	// 4. Iterates over in-memory Blockstore and writes proofs to the CAR
 	err = writer.writeProofs(ctx)
 	if err != nil {
-		return fmt.Errorf("failure writing proofs: %w", err)
+                return fmt.Errorf("share: failure writing proofs: %w", err)
 	}
 
 	return nil

From 016ab69243ec7a7ef9dd4c6815fe781dcc4ab33e Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Tue, 4 Oct 2022 16:12:27 +0200
Subject: [PATCH 11/18] fix: review fixes

---
 share/eds.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index 7e9c5dd96c..e7dc783197 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -40,26 +40,26 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	// 1. Reimport EDS. This is needed to traverse the NMT tree and cache the inner nodes (proofs)
 	writer, err := initializeWriter(ctx, eds, w)
 	if err != nil {
-                return fmt.Errorf("share: failure creating eds writer: %w", err)
+		return fmt.Errorf("share: failure creating eds writer: %w", err)
 	}
 
 	// 2. Creates and writes Carv1Header
 	//    - Roots are the eds Row + Col roots
 	err = writer.writeHeader()
 	if err != nil {
-                return fmt.Errorf("share: failure writing carv1 header: %w", err)
+		return fmt.Errorf("share: failure writing carv1 header: %w", err)
 	}
 
 	// 3. Iterates over shares in quadrant order vis eds.GetCell
-	err = writer.writeShares()
+	err = writer.writeQuadrants()
 	if err != nil {
-                return fmt.Errorf("share: failure writing shares: %w", err)
+		return fmt.Errorf("share: failure writing shares: %w", err)
 	}
 
 	// 4. Iterates over in-memory Blockstore and writes proofs to the CAR
 	err = writer.writeProofs(ctx)
 	if err != nil {
-                return fmt.Errorf("share: failure writing proofs: %w", err)
+		return fmt.Errorf("share: failure writing proofs: %w", err)
 	}
 
 	return nil
@@ -73,7 +73,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.
 	// shares are extracted from the eds so that we can reimport them to traverse
 	shares := ipld.ExtractEDS(eds)
 	if len(shares) == 0 {
-		return nil, fmt.Errorf("ipld: importing empty data")
+		return nil, fmt.Errorf("ipld: importing empty share")
 	}
 	// todo: add correct batch size here
 	squareSize := int(math.Sqrt(float64(len(shares))))
@@ -116,8 +116,8 @@ func (w *writingSession) writeHeader() error {
 	return nil
 }
 
-// writeShares reorders the shares to quadrant order and writes them to the CARv1 file.
-func (w *writingSession) writeShares() error {
+// writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file.
+func (w *writingSession) writeQuadrants() error {
 	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
 		// TODO: Okay. this is really weird and I don't understand:

From 3a27957f7df79f1d817f20175513161f6c506677 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Tue, 4 Oct 2022 16:18:29 +0200
Subject: [PATCH 12/18] refactor: resolving comments

---
 share/eds.go | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/share/eds.go b/share/eds.go
index e7dc783197..8c7627708b 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -73,7 +73,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.
 	// shares are extracted from the eds so that we can reimport them to traverse
 	shares := ipld.ExtractEDS(eds)
 	if len(shares) == 0 {
-		return nil, fmt.Errorf("ipld: importing empty share")
+		return nil, fmt.Errorf("share: importing empty data")
 	}
 	// todo: add correct batch size here
 	squareSize := int(math.Sqrt(float64(len(shares))))
@@ -120,9 +120,6 @@ func (w *writingSession) writeHeader() error {
 func (w *writingSession) writeQuadrants() error {
 	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
-		// TODO: Okay. this is really weird and I don't understand:
-		// We need to cut off the first byte like we do for inner nodes, but this share doesn't even have the prefix...
-		// So what is going on? If we don't do this, the cid doesn't match on read.
 		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share[1:]))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
@@ -147,7 +144,7 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 		if err != nil {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
-		// TODO: Learn why this doesn't match proofCid or node.Cid()
+		// we chop off the first byte, as it is an unnecessary type byte.
 		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()[1:]))
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)

From 94580cc6ce52e041c7db128984d6daff001f5ce0 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Fri, 7 Oct 2022 10:26:40 +0200
Subject: [PATCH 13/18] fix(share): removing type byte from leaf CIDs

---
 share/eds.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/eds.go b/share/eds.go
index 8c7627708b..352ea3bdcb 100644
--- a/share/eds.go
+++ b/share/eds.go
@@ -120,7 +120,7 @@ func (w *writingSession) writeHeader() error {
 func (w *writingSession) writeQuadrants() error {
 	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
-		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share[1:]))
+		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
 		}

From 0f7237d58f318e27a51c837d3c60fe8be7335bed Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Fri, 14 Oct 2022 06:30:55 +0200
Subject: [PATCH 14/18] chore: fixing imports

---
 share/{ => eds}/eds.go      | 17 ++++++++---------
 share/{ => eds}/eds_test.go | 11 ++++++-----
 share/ipld/nmt_adder.go     |  4 ++--
 3 files changed, 16 insertions(+), 16 deletions(-)
 rename share/{ => eds}/eds.go (93%)
 rename share/{ => eds}/eds_test.go (96%)

diff --git a/share/eds.go b/share/eds/eds.go
similarity index 93%
rename from share/eds.go
rename to share/eds/eds.go
index 352ea3bdcb..a9d2efda2c 100644
--- a/share/eds.go
+++ b/share/eds/eds.go
@@ -1,4 +1,4 @@
-package share
+package eds
 
 import (
 	"context"
@@ -18,9 +18,8 @@ import (
 	"github.com/ipld/go-car/util"
 	"github.com/tendermint/tendermint/pkg/wrapper"
 
-	"github.com/celestiaorg/celestia-node/ipld"
-	"github.com/celestiaorg/celestia-node/ipld/plugin"
-
+	"github.com/celestiaorg/celestia-node/share"
+	"github.com/celestiaorg/celestia-node/share/ipld"
 	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/rsmt2d"
 )
@@ -71,7 +70,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.
 	store := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
 	bs := blockservice.New(store, nil)
 	// shares are extracted from the eds so that we can reimport them to traverse
-	shares := ipld.ExtractEDS(eds)
+	shares := share.ExtractEDS(eds)
 	if len(shares) == 0 {
 		return nil, fmt.Errorf("share: importing empty data")
 	}
@@ -80,7 +79,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.
 	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize/2))
 	// this adder ignores leaves, so that they are not added to the store we iterate through in writeProofs
 	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
-	eds, err := rsmt2d.ImportExtendedDataSquare(shares, ipld.DefaultRSMT2DCodec(), tree.Constructor)
+	eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), tree.Constructor)
 	if err != nil {
 		return nil, fmt.Errorf("failure to recompute the extended data square: %w", err)
 	}
@@ -120,7 +119,7 @@ func (w *writingSession) writeHeader() error {
 func (w *writingSession) writeQuadrants() error {
 	shares := quadrantOrder(w.eds)
 	for _, share := range shares {
-		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
+		cid, err := ipld.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedLeaf(share))
 		if err != nil {
 			return fmt.Errorf("failure to get cid from share: %w", err)
 		}
@@ -145,7 +144,7 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
 		// we chop off the first byte, as it is an unnecessary type byte.
-		cid, err := plugin.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()[1:]))
+		cid, err := ipld.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()[1:]))
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)
 		}
@@ -206,7 +205,7 @@ func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
 	roots := append(eds.RowRoots(), eds.ColRoots()...)
 	rootCids := make([]cid.Cid, len(roots))
 	for i, r := range roots {
-		rootCids[i], err = plugin.CidFromNamespacedSha256(r)
+		rootCids[i], err = ipld.CidFromNamespacedSha256(r)
 		if err != nil {
 			return nil, fmt.Errorf("failure to get cid from root: %w", err)
 		}
diff --git a/share/eds_test.go b/share/eds/eds_test.go
similarity index 96%
rename from share/eds_test.go
rename to share/eds/eds_test.go
index 50ff313939..8617529092 100644
--- a/share/eds_test.go
+++ b/share/eds/eds_test.go
@@ -1,4 +1,4 @@
-package share
+package eds
 
 import (
 	"context"
@@ -8,12 +8,13 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carv1 "github.com/ipld/go-car"
+	"github.com/stretchr/testify/require"
 	"github.com/tendermint/tendermint/pkg/consts"
 
-	"github.com/celestiaorg/celestia-node/ipld"
-	"github.com/celestiaorg/rsmt2d"
+	"github.com/celestiaorg/celestia-node/share"
+	"github.com/celestiaorg/celestia-node/share/ipld"
 
-	"github.com/stretchr/testify/require"
+	"github.com/celestiaorg/rsmt2d"
 )
 
 func TestQuadrantOrder(t *testing.T) {
@@ -106,7 +107,7 @@ func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare {
 	f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE, 0600)
 	require.NoError(t, err, "error opening file")
 
-	eds := ipld.RandEDS(t, 4)
+	eds := share.RandEDS(t, 4)
 	err = WriteEDS(context.Background(), eds, f)
 	require.NoError(t, err, "error writing EDS to file")
 	f.Close()
diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 24b78c14ae..5a9854a2aa 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -54,12 +54,12 @@ func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
 		return // protect from further visits if there is an error
 	}
 
-	id := plugin.MustCidFromNamespacedSha256(hash)
+	id := MustCidFromNamespacedSha256(hash)
 	switch len(children) {
 	case 1:
 		break
 	case 2:
-		n.err = n.add.Add(n.ctx, plugin.NewNMTNode(id, children[0], children[1]))
+		n.err = n.add.Add(n.ctx, NewNMTNode(id, children[0], children[1]))
 	default:
 		panic("expected a binary tree")
 	}

From 1338108eebf0be0eef39e212d3af0119d34205dd Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Mon, 17 Oct 2022 09:12:11 +0200
Subject: [PATCH 15/18] fix: importing correct packages for nmt wrapper

---
 share/eds/eds.go      |  8 ++++----
 share/eds/eds_test.go | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/share/eds/eds.go b/share/eds/eds.go
index a9d2efda2c..0f298b93e8 100644
--- a/share/eds/eds.go
+++ b/share/eds/eds.go
@@ -6,8 +6,6 @@ import (
 	"io"
 	"math"
 
-	"github.com/tendermint/tendermint/pkg/consts"
-
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
@@ -16,7 +14,9 @@ import (
 	format "github.com/ipfs/go-ipld-format"
 	"github.com/ipld/go-car"
 	"github.com/ipld/go-car/util"
-	"github.com/tendermint/tendermint/pkg/wrapper"
+
+	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/wrapper"
 
 	"github.com/celestiaorg/celestia-node/share"
 	"github.com/celestiaorg/celestia-node/share/ipld"
@@ -193,7 +193,7 @@ func prependNamespace(quadrant int, share []byte) []byte {
 	case 0:
 		return append(share[:8], share...)
 	case 1, 2, 3:
-		return append(consts.ParitySharesNamespaceID, share...)
+		return append(appconsts.ParitySharesNamespaceID, share...)
 	default:
 		panic("invalid quadrant")
 	}
diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go
index 8617529092..755b9e41de 100644
--- a/share/eds/eds_test.go
+++ b/share/eds/eds_test.go
@@ -9,19 +9,19 @@ import (
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carv1 "github.com/ipld/go-car"
 	"github.com/stretchr/testify/require"
-	"github.com/tendermint/tendermint/pkg/consts"
+
+	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/celestiaorg/celestia-node/share"
 	"github.com/celestiaorg/celestia-node/share/ipld"
-
-	"github.com/celestiaorg/rsmt2d"
 )
 
 func TestQuadrantOrder(t *testing.T) {
 	// TODO: add more test cases
 	nID := []byte{0, 0, 0, 0, 0, 0, 0, 0}
-	parity := append(consts.ParitySharesNamespaceID, nID...) //nolint
-	doubleNID := append(nID, nID...)                         //nolint
+	parity := append(appconsts.ParitySharesNamespaceID, nID...) //nolint
+	doubleNID := append(nID, nID...)                            //nolint
 	result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{
 		append(nID, 1), append(nID, 2),
 		append(nID, 3), append(nID, 4),

From 8b8ccf3cac919554d5cdaa9a959ec7b39694a4aa Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Mon, 17 Oct 2022 13:37:00 +0200
Subject: [PATCH 16/18] chore: using newNmtNode from rebase

---
 share/ipld/nmt_adder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go
index 5a9854a2aa..6db890a228 100644
--- a/share/ipld/nmt_adder.go
+++ b/share/ipld/nmt_adder.go
@@ -59,7 +59,7 @@ func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) {
 	case 1:
 		break
 	case 2:
-		n.err = n.add.Add(n.ctx, NewNMTNode(id, children[0], children[1]))
+		n.err = n.add.Add(n.ctx, newNMTNode(id, append(children[0], children[1]...)))
 	default:
 		panic("expected a binary tree")
 	}

From 79c54099b4d55ee9049253541655271d582277d5 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Mon, 17 Oct 2022 13:47:42 +0200
Subject: [PATCH 17/18] fix: removing type byte after rebase

---
 share/eds/eds.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/share/eds/eds.go b/share/eds/eds.go
index 0f298b93e8..0becd654e9 100644
--- a/share/eds/eds.go
+++ b/share/eds/eds.go
@@ -144,7 +144,7 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
 		// we chop off the first byte, as it is an unnecessary type byte.
-		cid, err := ipld.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()[1:]))
+		cid, err := ipld.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()))
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)
 		}

From 854b60ca6b3160cf15a645a17240e56254b10347 Mon Sep 17 00:00:00 2001
From: Ryan <ryanford@poluglottos.com>
Date: Tue, 18 Oct 2022 10:05:46 +0200
Subject: [PATCH 18/18] fix: resolving review comments and correcting batch
 size

Co-authored-by: rene <41963722+renaynay@users.noreply.github.com>
---
 share/eds/eds.go      | 39 +++++++++++++++++++++++----------------
 share/eds/eds_test.go | 38 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 18 deletions(-)

diff --git a/share/eds/eds.go b/share/eds/eds.go
index 0becd654e9..90622220c4 100644
--- a/share/eds/eds.go
+++ b/share/eds/eds.go
@@ -2,6 +2,7 @@ package eds
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -24,6 +25,8 @@ import (
 	"github.com/celestiaorg/rsmt2d"
 )
 
+var ErrEmptySquare = errors.New("share: importing empty data")
+
 // writingSession contains the components needed to write an EDS to a CARv1 file with our custom node order.
 type writingSession struct {
 	eds *rsmt2d.ExtendedDataSquare
@@ -34,7 +37,8 @@ type writingSession struct {
 
 // WriteEDS writes the entire EDS into the given io.Writer as CARv1 file.
 // This includes all shares in quadrant order, followed by all inner nodes of the NMT tree.
-// Order: Carv1Header - Q1 - Q2 - Q3 - Q4 - InnerNodes
+// Order: [ Carv1Header | Q1 |  Q2 | Q3 | Q4 | inner nodes ]
+// For more information about the header: https://ipld.io/specs/transport/car/carv1/#header
 func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
 	// 1. Reimport EDS. This is needed to traverse the NMT tree and cache the inner nodes (proofs)
 	writer, err := initializeWriter(ctx, eds, w)
@@ -49,7 +53,7 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 		return fmt.Errorf("share: failure writing carv1 header: %w", err)
 	}
 
-	// 3. Iterates over shares in quadrant order vis eds.GetCell
+	// 3. Iterates over shares in quadrant order via eds.GetCell
 	err = writer.writeQuadrants()
 	if err != nil {
 		return fmt.Errorf("share: failure writing shares: %w", err)
@@ -60,7 +64,6 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer)
 	if err != nil {
 		return fmt.Errorf("share: failure writing proofs: %w", err)
 	}
-
 	return nil
 }
 
@@ -71,14 +74,15 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.
 	bs := blockservice.New(store, nil)
 	// shares are extracted from the eds so that we can reimport them to traverse
 	shares := share.ExtractEDS(eds)
-	if len(shares) == 0 {
-		return nil, fmt.Errorf("share: importing empty data")
+	shareCount := len(shares)
+	if shareCount == 0 {
+		return nil, ErrEmptySquare
 	}
-	// todo: add correct batch size here
-	squareSize := int(math.Sqrt(float64(len(shares))))
-	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(squareSize/2))
+	odsWidth := int(math.Sqrt(float64(shareCount)) / 2)
+	// (shareCount*2) - (odsWidth*4) is the amount of inner nodes visited
+	batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(innerNodeBatchSize(shareCount, odsWidth)))
 	// this adder ignores leaves, so that they are not added to the store we iterate through in writeProofs
-	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(odsWidth), nmt.NodeVisitor(batchAdder.VisitInnerNodes))
 	eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), tree.Constructor)
 	if err != nil {
 		return nil, fmt.Errorf("failure to recompute the extended data square: %w", err)
@@ -105,14 +109,10 @@ func (w *writingSession) writeHeader() error {
 		return fmt.Errorf("failure getting root cids: %w", err)
 	}
 
-	err = car.WriteHeader(&car.CarHeader{
+	return car.WriteHeader(&car.CarHeader{
 		Roots:   rootCids,
 		Version: 1,
 	}, w.w)
-	if err != nil {
-		return fmt.Errorf("failure writing carv1 header: %w", err)
-	}
-	return nil
 }
 
 // writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file.
@@ -143,7 +143,6 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 		if err != nil {
 			return fmt.Errorf("failure to get proof from the blockstore: %w", err)
 		}
-		// we chop off the first byte, as it is an unnecessary type byte.
 		cid, err := ipld.CidFromNamespacedSha256(nmt.Sha256Namespace8FlaggedInner(node.RawData()))
 		if err != nil {
 			return fmt.Errorf("failure to get cid: %w", err)
@@ -156,7 +155,9 @@ func (w *writingSession) writeProofs(ctx context.Context) error {
 	return nil
 }
 
-// quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, adding the wrapped namespace
+// quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, prepending the respective namespace
+// to the shares.
+// e.g. [ Q1 R1 | Q1 R2 | Q1 R3 | Q1 R4 | Q2 R1 | Q2 R2 .... ]
 func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
 	size := eds.Width() * eds.Width()
 	shares := make([][]byte, size)
@@ -212,3 +213,9 @@ func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
 	}
 	return rootCids, nil
 }
+
+// innerNodeBatchSize calculates the total number of inner nodes in an EDS,
+// to be flushed to the dagstore in a single write.
+func innerNodeBatchSize(shareCount int, odsWidth int) int {
+	return (shareCount * 2) - (odsWidth * 4)
+}
diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go
index 755b9e41de..00552b475b 100644
--- a/share/eds/eds_test.go
+++ b/share/eds/eds_test.go
@@ -8,6 +8,7 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carv1 "github.com/ipld/go-car"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
@@ -75,7 +76,9 @@ func TestWriteEDSIncludesRoots(t *testing.T) {
 	defer f.Close()
 
 	bs := blockstore.NewBlockstore(ds.NewMapDatastore())
-	loaded, err := carv1.LoadCar(context.Background(), bs, f)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	loaded, err := carv1.LoadCar(ctx, bs, f)
 	require.NoError(t, err, "error loading car file")
 	for _, root := range loaded.Roots {
 		ok, err := bs.Has(context.Background(), root)
@@ -100,7 +103,36 @@ func TestWriteEDSInQuadrantOrder(t *testing.T) {
 	}
 }
 
+// TestInnerNodeBatchSize verifies that the number of unique inner nodes is equal to ipld.BatchSize - shareCount.
+func TestInnerNodeBatchSize(t *testing.T) {
+	tests := []struct {
+		name      string
+		origWidth int
+	}{
+		{"2", 2},
+		{"4", 4},
+		{"8", 8},
+		{"16", 16},
+		{"32", 32},
+		// {"64", 64}, // test case too large for CI with race detector
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			extendedWidth := tt.origWidth * 2
+			shareCount := extendedWidth * extendedWidth
+			assert.Equalf(
+				t,
+				innerNodeBatchSize(shareCount, tt.origWidth),
+				ipld.BatchSize(extendedWidth)-shareCount,
+				"batchSize(%v)", extendedWidth,
+			)
+		})
+	}
+}
+
 func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
 	tmpDir := t.TempDir()
 	err := os.Chdir(tmpDir)
 	require.NoError(t, err, "error changing to the temporary test directory")
@@ -108,13 +140,15 @@ func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare {
 	require.NoError(t, err, "error opening file")
 
 	eds := share.RandEDS(t, 4)
-	err = WriteEDS(context.Background(), eds, f)
+	err = WriteEDS(ctx, eds, f)
 	require.NoError(t, err, "error writing EDS to file")
+	t.Cleanup(cancel)
 	f.Close()
 	return eds
 }
 
 func openWrittenEDS(t *testing.T) *os.File {
+	t.Helper()
 	f, err := os.OpenFile("test.car", os.O_RDONLY, 0600)
 	require.NoError(t, err, "error opening file")
 	return f