tools: new endpoints for block generator

tools/block-generator/generator/generate.go

@@ -17,7 +17,6 @@
 package generator
 
 import (
-	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,6 +25,9 @@ import (
 	"time"
 
 	cconfig "github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 
 	"github.com/algorand/go-algorand/agreement"
@@ -133,6 +135,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
 	gen.genesisHash[31] = 3
 
 	gen.initializeAccounting()
+	gen.initializeLedger()
 
 	for _, val := range getTransactionOptions() {
 		switch val {
@@ -177,7 +180,9 @@ type Generator interface {
 	WriteBlock(output io.Writer, round uint64) error
 	WriteAccount(output io.Writer, accountString string) error
 	WriteStatus(output io.Writer) error
+	WriteDeltas(output io.Writer, round uint64) error
 	Accounts() <-chan basics.Address
+	Stop()
 }
 
 type generator struct {
@@ -224,6 +229,9 @@ type generator struct {
 
 	// Reporting information from transaction type to data
 	reportData Report
+
+	//	ledger
+	ledger *ledger.Ledger
 }
 
 type assetData struct {
@@ -410,20 +418,25 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 	if err != nil {
 		return err
 	}
-
+	g.ledger.AddBlock(cert.Block, agreement.Certificate{})
+	g.ledger.WaitForCommit(basics.Round(g.round))
 	g.finishRound(numTxnForBlock)
 	return nil
 }
 
-func indexToAccount(i uint64) (addr basics.Address) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	binary.LittleEndian.PutUint64(addr[:], i+1)
-	return
-}
-
-func accountToIndex(a basics.Address) (addr uint64) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	return binary.LittleEndian.Uint64(a[:]) - 1
+// WriteDeltas generates returns the deltas for payset.
+func (g *generator) WriteDeltas(output io.Writer, round uint64) error {
+	delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round))
+	if err != nil {
+		return fmt.Errorf("err getting state delta for round %d, %v", round, err)
+	}
+	// msgp encode deltas
+	data, err := encode(protocol.CodecHandle, delta)
+	_, err = output.Write(data)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // initializeAccounting creates the genesis accounts.
@@ -686,6 +699,35 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S
 	return signTxn(txn), transactions.ApplyData{}, nil
 }
 
+func (g *generator) initializeLedger() {
+	genBal := convertToGenesisBalances(g.balances)
+	// add rewards pool with min balance
+	genBal[g.rewardsPool] = basics.AccountData{
+		MicroAlgos: basics.MicroAlgos{g.params.MinBalance},
+	}
+	bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool)
+	block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash)
+	if err != nil {
+		fmt.Printf("error making genesis: %v\n.", err)
+		os.Exit(1)
+	}
+	l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{
+		Block:       block,
+		Accounts:    bal.Balances,
+		GenesisHash: g.genesisHash,
+	}, cconfig.GetDefaultLocal())
+	if err != nil {
+		fmt.Printf("error initializing ledger: %v\n.", err)
+		os.Exit(1)
+	}
+	g.ledger = l
+}
+
+// Stop cleans up allocated resources.
+func (g *generator) Stop() {
+	g.ledger.Close()
+}
+
 func (g *generator) WriteAccount(output io.Writer, accountString string) error {
 	addr, err := basics.UnmarshalChecksumAddress(accountString)
 	if err != nil {

tools/block-generator/generator/generate_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/rpcs"
@@ -31,7 +32,7 @@ func makePrivateGenerator(t *testing.T) *generator {
 	partitiontest.PartitionTest(t)
 	publicGenerator, err := MakeGenerator(GenerationConfig{
 		NumGenesisAccounts:           10,
-		GenesisAccountInitialBalance: 10000000000000000000,
+		GenesisAccountInitialBalance: 1000000000000,
 		PaymentTransactionFraction:   1.0,
 		PaymentNewAccountFraction:    1.0,
 		AssetCreateFraction:          1.0,
@@ -204,6 +205,9 @@ func TestWriteRound(t *testing.T) {
 	var block rpcs.EncodedBlockCert
 	protocol.Decode(data, &block)
 	require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock))
+	require.Equal(t, basics.Round(1), g.ledger.Latest())
+	_, err := g.ledger.GetStateDeltaForRound(1)
+	require.NoError(t, err)
 }
 
 func TestIndexToAccountAndAccountToIndex(t *testing.T) {

tools/block-generator/generator/server.go

@@ -20,10 +20,12 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/algorand/go-algorand/tools/block-generator/util"
+	"github.com/gorilla/mux"
 	"gopkg.in/yaml.v3"
 )
 
@@ -60,17 +62,19 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B
 	gen, err := MakeGenerator(config)
 	util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile)
 
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", help)
-	mux.Handle("/v2/blocks/", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen))))
-	mux.HandleFunc("/v2/accounts/", getAccountHandler(gen))
-	mux.HandleFunc("/genesis", getGenesisHandler(gen))
-	mux.HandleFunc("/report", getReportHandler(gen))
-	mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen))
+	r := mux.NewRouter()
+	r.HandleFunc("/", help)
+	r.Handle("/v2/blocks/{round}", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen))))
+	r.HandleFunc("/v2/accounts/", getAccountHandler(gen))
+	r.HandleFunc("/genesis", getGenesisHandler(gen))
+	r.HandleFunc("/report", getReportHandler(gen))
+	r.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen))
+	r.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {})
+	r.HandleFunc("/v2/deltas/{round}", getDeltasHandler(gen))
 
 	return &http.Server{
 		Addr:              addr,
-		Handler:           mux,
+		Handler:           r,
 		ReadHeaderTimeout: 3 * time.Second,
 	}, gen
 }
@@ -107,7 +111,13 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
 func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// The generator doesn't actually care about the block...
-		round, err := parseRound(r.URL.Path)
+		vars := mux.Vars(r)
+		param, ok := vars["round"]
+		if !ok {
+			http.Error(w, "round missing", http.StatusBadRequest)
+			return
+		}
+		round, err := strconv.ParseUint(param, 10, 64)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
@@ -130,35 +140,26 @@ func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
 	}
 }
 
-const blockQueryPrefix = "/v2/blocks/"
-const blockQueryBlockIdx = len(blockQueryPrefix)
-const accountsQueryPrefix = "/v2/accounts/"
-const accountsQueryAccountIdx = len(accountsQueryPrefix)
-
-func parseRound(path string) (uint64, error) {
-	if !strings.HasPrefix(path, blockQueryPrefix) {
-		return 0, fmt.Errorf("not a blocks query: %s", path)
-	}
-
-	result := uint64(0)
-	pathlen := len(path)
-
-	if pathlen == blockQueryBlockIdx {
-		return 0, fmt.Errorf("no block in path")
-	}
-
-	for i := blockQueryBlockIdx; i < pathlen; i++ {
-		if path[i] < '0' || path[i] > '9' {
-			if i == blockQueryBlockIdx {
-				return 0, fmt.Errorf("no block in path")
-			}
-			break
+func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		rd, ok := vars["round"]
+		if !ok {
+			http.Error(w, "round missing", http.StatusBadRequest)
+			return
+		}
+		round, err := strconv.ParseUint(rd, 10, 64)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
 		}
-		result = (uint64(10) * result) + uint64(int(path[i])-'0')
+		maybeWriteError(w, gen.WriteDeltas(w, round))
 	}
-	return result, nil
 }
 
+const accountsQueryPrefix = "/v2/accounts/"
+const accountsQueryAccountIdx = len(accountsQueryPrefix)
+
 func parseAccount(path string) (string, error) {
 	if !strings.HasPrefix(path, accountsQueryPrefix) {
 		return "", fmt.Errorf("not a accounts query: %s", path)

tools/block-generator/generator/server_test.go

@@ -17,13 +17,10 @@
 package generator
 
 import (
-	"fmt"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/algorand/go-algorand/test/partitiontest"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,65 +41,3 @@ func TestInitConfigFileNotExist(t *testing.T) {
 		require.Fail(t, "This should generate a path error")
 	}
 }
-
-func TestParseRound(t *testing.T) {
-	partitiontest.PartitionTest(t)
-	var testcases = []struct {
-		name          string
-		url           string
-		expectedRound uint64
-		err           string
-	}{
-		{
-			name:          "no block",
-			url:           "/v2/blocks/",
-			expectedRound: 0,
-			err:           "no block in path",
-		},
-		{
-			name:          "no block 2",
-			url:           "/v2/blocks/?nothing",
-			expectedRound: 0,
-			err:           "no block in path",
-		},
-		{
-			name:          "invalid prefix",
-			url:           "/v2/wrong/prefix/1",
-			expectedRound: 0,
-			err:           "not a blocks query",
-		},
-		{
-			name:          "normal one digit",
-			url:           fmt.Sprintf("%s1", blockQueryPrefix),
-			expectedRound: 1,
-			err:           "",
-		},
-		{
-			name:          "normal long number",
-			url:           fmt.Sprintf("%s12345678", blockQueryPrefix),
-			expectedRound: 12345678,
-			err:           "",
-		},
-		{
-			name:          "with query parameters",
-			url:           fmt.Sprintf("%s1234?pretty", blockQueryPrefix),
-			expectedRound: 1234,
-			err:           "",
-		},
-	}
-
-	for _, testcase := range testcases {
-		t.Run(testcase.name, func(t *testing.T) {
-			round, err := parseRound(testcase.url)
-			if len(testcase.err) == 0 {
-				msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%d' received error: %v",
-					testcase.url, testcase.expectedRound, err)
-				require.NoError(t, err, msg)
-				assert.Equal(t, testcase.expectedRound, round)
-			} else {
-				require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err))
-				require.True(t, strings.Contains(err.Error(), testcase.err))
-			}
-		})
-	}
-}

tools/block-generator/generator/utils.go

@@ -17,8 +17,12 @@
 package generator
 
 import (
+	"encoding/binary"
 	"fmt"
 	"math/rand"
+
+	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-codec/codec"
 )
 
 func weightedSelection(weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) {
@@ -43,3 +47,45 @@ func weightedSelectionInternal(selectionNumber float32, weights []float32, optio
 	selection = defaultOption
 	return
 }
+
+func indexToAccount(i uint64) (addr basics.Address) {
+	// Make sure we don't generate a zero address by adding 1 to i
+	binary.LittleEndian.PutUint64(addr[:], i+1)
+	return
+}
+
+func accountToIndex(a basics.Address) (addr uint64) {
+	// Make sure we don't generate a zero address by adding 1 to i
+	return binary.LittleEndian.Uint64(a[:]) - 1
+}
+
+func convertToGenesisBalances(balances []uint64) map[basics.Address]basics.AccountData {
+	genesisBalances := make(map[basics.Address]basics.AccountData)
+	for i, balance := range balances {
+		genesisBalances[indexToAccount(uint64(i))] = basics.AccountData{
+			MicroAlgos: basics.MicroAlgos{Raw: balance},
+		}
+	}
+	return genesisBalances
+}
+
+func encode(handle codec.Handle, obj interface{}) ([]byte, error) {
+	var output []byte
+	enc := codec.NewEncoderBytes(&output, handle)
+
+	err := enc.Encode(obj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode object: %v", err)
+	}
+	return output, nil
+}
+
+func decode(handle codec.Handle, data []byte, v interface{}) error {
+	enc := codec.NewDecoderBytes(data, handle)
+
+	err := enc.Decode(v)
+	if err != nil {
+		return fmt.Errorf("failed to decode object: %v", err)
+	}
+	return nil
+}