Swing-store export data outside of genesis file (#9549)

Closes #9567

## Description
Stores the swing-store export data outside of the genesis file, and only include a hash of it in genesis for validation.

This is the bulk of the data in the genesis file (on mainnet 6GB vs 300MB for the rest), and it serves little purpose to keep it in there. Furthermore golevelDB chokes when cosmos attempts to store the genesis file inside the DB (limit of 4GB documents)

### Security Considerations
None, the data remains validated

### Scaling Considerations
Reduces memory usage on genesis export / import. Furthermore we avoid iterating over the data twice on import, which is painfully slow for IAVL.

### Documentation Considerations
None

### Testing Considerations
Added a3p acceptance test

Manually tested as follow:
```
cd packages/cosmic-swingset
make scenario2-setup
make scenario2-run-chain
# wait until there are blocks, then kill
mkdir t1/n0/export
agd --home t1/n0 export --export-dir t1/n0/export
agd --home t1/n0 tendermint unsafe-reset-all
mv t1/n0/export/* t1/n0/config
make scenario2-run-chain
# verify blocks are being produced after genesis restart
```

### Upgrade Considerations
There is a compatibility mode to load genesis files with export data embedded.

a3p-integration/proposals/z:acceptance/genesis-test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+source /usr/src/upgrade-test-scripts/env_setup.sh
+
+export_genesis() {
+  GENESIS_EXPORT_DIR="$1"
+  shift
+  GENESIS_HEIGHT_ARG=
+
+  if [ -n "$1" ]; then
+    GENESIS_HEIGHT_ARG="--height $1"
+    shift
+  fi
+
+  agd export --export-dir "$GENESIS_EXPORT_DIR" $GENESIS_HEIGHT_ARG "$@"
+}
+
+killAgd
+FORK_TEST_DIR="$(mktemp -t -d fork-test-XXX)"
+mkdir -p "$FORK_TEST_DIR/config"
+cp /root/.agoric/config/priv_validator_key.json "$FORK_TEST_DIR/config"
+agd --home "$FORK_TEST_DIR" tendermint unsafe-reset-all
+
+export_genesis "$FORK_TEST_DIR/config"
+startAgd --home "$FORK_TEST_DIR"

a3p-integration/proposals/z:acceptance/test.sh

@@ -13,3 +13,4 @@ yarn ava ./*.test.js
 ./create-kread-item-test.sh
 
 ./state-sync-snapshots-test.sh
+./genesis-test.sh

golang/cosmos/proto/agoric/swingset/genesis.proto

@@ -17,6 +17,10 @@ message GenesisState {
     repeated SwingStoreExportDataEntry swing_store_export_data = 4 [
         (gogoproto.jsontag)    = "swingStoreExportData"
     ];
+
+    string swing_store_export_data_hash = 5 [
+        (gogoproto.jsontag)    = "swingStoreExportDataHash"
+    ];
 }
 
 // A SwingStore "export data" entry.

golang/cosmos/types/kv_entry_helpers.go

@@ -232,3 +232,45 @@ func EncodeKVEntryReaderToJsonl(reader KVEntryReader, bytesWriter io.Writer) (er
 		}
 	}
 }
+
+var _ KVEntryReader = &kvHookingReader{}
+
+// kvHookingReader is a KVEntryReader backed by another KVEntryReader which
+// provides callbacks for read and close
+type kvHookingReader struct {
+	reader  KVEntryReader
+	onRead  func(entry KVEntry) error
+	onClose func() error
+}
+
+// NewKVHookingReader returns a KVEntryReader backed by another KVEntryReader
+func NewKVHookingReader(reader KVEntryReader, onRead func(entry KVEntry) error, onClose func() error) KVEntryReader {
+	return &kvHookingReader{
+		reader,
+		onRead,
+		onClose,
+	}
+}
+
+// Read yields the next KVEntry from the source reader
+// Implements KVEntryReader
+func (hr kvHookingReader) Read() (next KVEntry, err error) {
+	next, err = hr.reader.Read()
+
+	if err == nil {
+		err = hr.onRead(next)
+	}
+
+	return next, err
+}
+
+// Close releases the underlying source reader
+// Implements KVEntryReader
+func (hr kvHookingReader) Close() error {
+	err := hr.reader.Close()
+	if err == nil {
+		err = hr.onClose()
+	}
+
+	return err
+}

golang/cosmos/x/swingset/genesis.go

@@ -2,7 +2,13 @@ package swingset
 
 import (
 	// "os"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"hash"
+	"strings"
 
 	agoric "github.com/Agoric/agoric-sdk/golang/cosmos/types"
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/keeper"
@@ -35,8 +41,10 @@ func InitGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingStore
 	k.SetState(ctx, data.GetState())
 
 	swingStoreExportData := data.GetSwingStoreExportData()
-	if len(swingStoreExportData) == 0 {
+	if len(swingStoreExportData) == 0 && data.SwingStoreExportDataHash == "" {
 		return true
+	} else if data.SwingStoreExportDataHash != "" && len(swingStoreExportData) > 0 {
+		panic("Swingset genesis state cannot have both export data and hash of export data")
 	}
 
 	artifactProvider, err := keeper.OpenSwingStoreExportDirectory(swingStoreExportDir)
@@ -46,15 +54,62 @@ func InitGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingStore
 
 	swingStore := k.GetSwingStore(ctx)
 
-	for _, entry := range swingStoreExportData {
-		swingStore.Set([]byte(entry.Key), []byte(entry.Value))
-	}
-
 	snapshotHeight := uint64(ctx.BlockHeight())
 
-	getExportDataReader := func() (agoric.KVEntryReader, error) {
-		exportDataIterator := swingStore.Iterator(nil, nil)
-		return agoric.NewKVIteratorReader(exportDataIterator), nil
+	var getExportDataReader func() (agoric.KVEntryReader, error)
+
+	if len(swingStoreExportData) > 0 {
+		for _, entry := range swingStoreExportData {
+			swingStore.Set([]byte(entry.Key), []byte(entry.Value))
+		}
+		getExportDataReader = func() (agoric.KVEntryReader, error) {
+			exportDataIterator := swingStore.Iterator(nil, nil)
+			return agoric.NewKVIteratorReader(exportDataIterator), nil
+		}
+	} else {
+		hashParts := strings.SplitN(data.SwingStoreExportDataHash, ":", 2)
+		if len(hashParts) != 2 {
+			panic(fmt.Errorf("invalid swing-store export data hash %s", data.SwingStoreExportDataHash))
+		}
+		if hashParts[0] != "sha256" {
+			panic(fmt.Errorf("invalid swing-store export data hash algorithm %s, expected sha256", hashParts[0]))
+		}
+		sha256Hash, err := hex.DecodeString(hashParts[1])
+		if err != nil {
+			panic(err)
+		}
+		getExportDataReader = func() (agoric.KVEntryReader, error) {
+			kvReader, err := artifactProvider.GetExportDataReader()
+			if err != nil {
+				return nil, err
+			}
+
+			if kvReader == nil {
+				return nil, fmt.Errorf("swing-store export has no export data")
+			}
+
+			hasher := sha256.New()
+			encoder := json.NewEncoder(hasher)
+			encoder.SetEscapeHTML(false)
+
+			return agoric.NewKVHookingReader(kvReader, func(entry agoric.KVEntry) error {
+				key := []byte(entry.Key())
+
+				if !entry.HasValue() {
+					swingStore.Delete(key)
+				} else {
+					swingStore.Set(key, []byte(entry.StringValue()))
+				}
+
+				return encoder.Encode(entry)
+			}, func() error {
+				sum := hasher.Sum(nil)
+				if !bytes.Equal(sum, sha256Hash) {
+					return fmt.Errorf("swing-store data sha256sum didn't match. expected %x, got %x", sha256Hash, sum)
+				}
+				return nil
+			}), nil
+		}
 	}
 
 	err = swingStoreExportsHandler.RestoreExport(
@@ -79,25 +134,17 @@ func ExportGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingSto
 	gs := &types.GenesisState{
 		Params:               k.GetParams(ctx),
 		State:                k.GetState(ctx),
-		SwingStoreExportData: []*types.SwingStoreExportDataEntry{},
-	}
-
-	exportDataIterator := k.GetSwingStore(ctx).Iterator(nil, nil)
-	defer exportDataIterator.Close()
-	for ; exportDataIterator.Valid(); exportDataIterator.Next() {
-		entry := types.SwingStoreExportDataEntry{
-			Key:   string(exportDataIterator.Key()),
-			Value: string(exportDataIterator.Value()),
-		}
-		gs.SwingStoreExportData = append(gs.SwingStoreExportData, &entry)
+		SwingStoreExportData: nil,
 	}
 
 	snapshotHeight := uint64(ctx.BlockHeight())
 
+	eventHandler := swingStoreGenesisEventHandler{exportDir: swingStoreExportDir, snapshotHeight: snapshotHeight, swingStore: k.GetSwingStore(ctx), hasher: sha256.New()}
+
 	err := swingStoreExportsHandler.InitiateExport(
 		// The export will fail if the export of a historical height was requested
 		snapshotHeight,
-		swingStoreGenesisEventHandler{exportDir: swingStoreExportDir, snapshotHeight: snapshotHeight},
+		eventHandler,
 		keeper.SwingStoreExportOptions{
 			ArtifactMode:   keeper.SwingStoreArtifactModeOperational,
 			ExportDataMode: keeper.SwingStoreExportDataModeSkip,
@@ -112,12 +159,16 @@ func ExportGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingSto
 		panic(err)
 	}
 
+	gs.SwingStoreExportDataHash = fmt.Sprintf("sha256:%x", eventHandler.hasher.Sum(nil))
+
 	return gs
 }
 
 type swingStoreGenesisEventHandler struct {
 	exportDir      string
 	snapshotHeight uint64
+	swingStore     sdk.KVStore
+	hasher         hash.Hash
 }
 
 func (eventHandler swingStoreGenesisEventHandler) OnExportStarted(height uint64, retrieveSwingStoreExport func() error) error {
@@ -131,7 +182,17 @@ func (eventHandler swingStoreGenesisEventHandler) OnExportRetrieved(provider kee
 
 	artifactsProvider := keeper.SwingStoreExportProvider{
 		GetExportDataReader: func() (agoric.KVEntryReader, error) {
-			return nil, nil
+			exportDataIterator := eventHandler.swingStore.Iterator(nil, nil)
+			kvReader := agoric.NewKVIteratorReader(exportDataIterator)
+			eventHandler.hasher.Reset()
+			encoder := json.NewEncoder(eventHandler.hasher)
+			encoder.SetEscapeHTML(false)
+
+			return agoric.NewKVHookingReader(kvReader, func(entry agoric.KVEntry) error {
+				return encoder.Encode(entry)
+			}, func() error {
+				return nil
+			}), nil
 		},
 		ReadNextArtifact: provider.ReadNextArtifact,
 	}

golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go

@@ -787,7 +787,18 @@ func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStore
 // a jsonl-like file, before saving the export manifest linking these together.
 // The export manifest filename and overall export format is common with the JS
 // swing-store import/export logic.
-func WriteSwingStoreExportToDirectory(provider SwingStoreExportProvider, exportDir string) error {
+func WriteSwingStoreExportToDirectory(provider SwingStoreExportProvider, exportDir string) (err error) {
+	handleDeferError := func(fn func() error) {
+		deferError := fn()
+		if err == nil {
+			err = deferError
+		} else if deferError != nil {
+			// Safe to wrap error and use detailed error info since this error
+			// will not go back into swingset layers
+			err = sdkioerrors.Wrapf(err, "deferred error %+v", deferError)
+		}
+	}
+
 	manifest := exportManifest{
 		BlockHeight: provider.BlockHeight,
 	}
@@ -798,14 +809,14 @@ func WriteSwingStoreExportToDirectory(provider SwingStoreExportProvider, exportD
 	}
 
 	if exportDataReader != nil {
-		defer exportDataReader.Close()
+		defer handleDeferError(exportDataReader.Close)
 
 		manifest.Data = exportDataFilename
 		exportDataFile, err := os.OpenFile(filepath.Join(exportDir, exportDataFilename), os.O_CREATE|os.O_WRONLY, exportedFilesMode)
 		if err != nil {
 			return err
 		}
-		defer exportDataFile.Close()
+		defer handleDeferError(exportDataFile.Close)
 
 		err = agoric.EncodeKVEntryReaderToJsonl(exportDataReader, exportDataFile)
 		if err != nil {