tools: new endpoints for block generator (#5257)

tools/block-generator/generator/generate.go

@@ -17,7 +17,6 @@
 package generator
 
 import (
-	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,6 +25,9 @@ import (
 	"time"
 
 	cconfig "github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 
 	"github.com/algorand/go-algorand/agreement"
@@ -133,6 +135,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
 	gen.genesisHash[31] = 3
 
 	gen.initializeAccounting()
+	gen.initializeLedger()
 
 	for _, val := range getTransactionOptions() {
 		switch val {
@@ -177,7 +180,9 @@ type Generator interface {
 	WriteBlock(output io.Writer, round uint64) error
 	WriteAccount(output io.Writer, accountString string) error
 	WriteStatus(output io.Writer) error
+	WriteDeltas(output io.Writer, round uint64) error
 	Accounts() <-chan basics.Address
+	Stop()
 }
 
 type generator struct {
@@ -224,6 +229,9 @@ type generator struct {
 
 	// Reporting information from transaction type to data
 	reportData Report
+
+	// ledger
+	ledger *ledger.Ledger
 }
 
 type assetData struct {
@@ -410,20 +418,31 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 	if err != nil {
 		return err
 	}
-
+	err = g.ledger.AddBlock(cert.Block, agreement.Certificate{})
+	if err != nil {
+		return err
+	}
+	g.ledger.WaitForCommit(basics.Round(g.round))
 	g.finishRound(numTxnForBlock)
 	return nil
 }
 
-func indexToAccount(i uint64) (addr basics.Address) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	binary.LittleEndian.PutUint64(addr[:], i+1)
-	return
-}
-
-func accountToIndex(a basics.Address) (addr uint64) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	return binary.LittleEndian.Uint64(a[:]) - 1
+// WriteDeltas generates returns the deltas for payset.
+func (g *generator) WriteDeltas(output io.Writer, round uint64) error {
+	delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round))
+	if err != nil {
+		return fmt.Errorf("err getting state delta for round %d, %v", round, err)
+	}
+	// msgp encode deltas
+	data, err := encode(protocol.CodecHandle, delta)
+	if err != nil {
+		return err
+	}
+	_, err = output.Write(data)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // initializeAccounting creates the genesis accounts.
@@ -686,6 +705,35 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S
 	return signTxn(txn), transactions.ApplyData{}, nil
 }
 
+func (g *generator) initializeLedger() {
+	genBal := convertToGenesisBalances(g.balances)
+	// add rewards pool with min balance
+	genBal[g.rewardsPool] = basics.AccountData{
+		MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance},
+	}
+	bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool)
+	block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash)
+	if err != nil {
+		fmt.Printf("error making genesis: %v\n.", err)
+		os.Exit(1)
+	}
+	l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{
+		Block:       block,
+		Accounts:    bal.Balances,
+		GenesisHash: g.genesisHash,
+	}, cconfig.GetDefaultLocal())
+	if err != nil {
+		fmt.Printf("error initializing ledger: %v\n.", err)
+		os.Exit(1)
+	}
+	g.ledger = l
+}
+
+// Stop cleans up allocated resources.
+func (g *generator) Stop() {
+	g.ledger.Close()
+}
+
 func (g *generator) WriteAccount(output io.Writer, accountString string) error {
 	addr, err := basics.UnmarshalChecksumAddress(accountString)
 	if err != nil {

tools/block-generator/generator/generate_test.go

@@ -18,8 +18,11 @@ package generator
 
 import (
 	"bytes"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/rpcs"
@@ -31,7 +34,7 @@ func makePrivateGenerator(t *testing.T) *generator {
 	partitiontest.PartitionTest(t)
 	publicGenerator, err := MakeGenerator(GenerationConfig{
 		NumGenesisAccounts:           10,
-		GenesisAccountInitialBalance: 10000000000000000000,
+		GenesisAccountInitialBalance: 1000000000000,
 		PaymentTransactionFraction:   1.0,
 		PaymentNewAccountFraction:    1.0,
 		AssetCreateFraction:          1.0,
@@ -204,13 +207,45 @@ func TestWriteRound(t *testing.T) {
 	var block rpcs.EncodedBlockCert
 	protocol.Decode(data, &block)
 	require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock))
+	require.NotNil(t, g.ledger)
+	require.Equal(t, basics.Round(1), g.ledger.Latest())
+	_, err := g.ledger.GetStateDeltaForRound(1)
+	require.NoError(t, err)
 }
 
-func TestIndexToAccountAndAccountToIndex(t *testing.T) {
+func TestHandlers(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	for i := uint64(0); i < uint64(100000); i++ {
-		acct := indexToAccount(i)
-		result := accountToIndex(acct)
-		require.Equal(t, i, result)
+	g := makePrivateGenerator(t)
+	handler := getBlockHandler(g)
+	var testcases = []struct {
+		name string
+		url  string
+		err  string
+	}{
+		{
+			name: "no block",
+			url:  "/v2/blocks/?nothing",
+			err:  "invalid request path, /",
+		},
+		{
+			name: "blocks: round must be numeric",
+			url:  "/v2/blocks/round",
+			err:  `strconv.ParseUint: parsing "round": invalid syntax`,
+		},
+		{
+			name: "deltas: round must be numeric",
+			url:  "/v2/deltas/round",
+			err:  `strconv.ParseUint: parsing "round": invalid syntax`,
+		},
+	}
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", testcase.url, nil)
+			w := httptest.NewRecorder()
+			handler(w, req)
+			require.Equal(t, http.StatusBadRequest, w.Code)
+			require.Contains(t, w.Body.String(), testcase.err)
+		})
 	}
 }

tools/block-generator/generator/server.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -67,6 +68,8 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B
 	mux.HandleFunc("/genesis", getGenesisHandler(gen))
 	mux.HandleFunc("/report", getReportHandler(gen))
 	mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen))
+	mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {})
+	mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen))
 
 	return &http.Server{
 		Addr:              addr,
@@ -107,68 +110,55 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
 func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// The generator doesn't actually care about the block...
-		round, err := parseRound(r.URL.Path)
+		s, err := parseURL(r.URL.Path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		round, err := strconv.ParseUint(s, 0, 64)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-
 		maybeWriteError(w, gen.WriteBlock(w, round))
 	}
 }
 
 func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// The generator doesn't actually care about the block...
-		account, err := parseAccount(r.URL.Path)
+		account, err := parseURL(r.URL.Path)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-
 		maybeWriteError(w, gen.WriteAccount(w, account))
 	}
 }
 
-const blockQueryPrefix = "/v2/blocks/"
-const blockQueryBlockIdx = len(blockQueryPrefix)
-const accountsQueryPrefix = "/v2/accounts/"
-const accountsQueryAccountIdx = len(accountsQueryPrefix)
-
-func parseRound(path string) (uint64, error) {
-	if !strings.HasPrefix(path, blockQueryPrefix) {
-		return 0, fmt.Errorf("not a blocks query: %s", path)
-	}
-
-	result := uint64(0)
-	pathlen := len(path)
-
-	if pathlen == blockQueryBlockIdx {
-		return 0, fmt.Errorf("no block in path")
-	}
-
-	for i := blockQueryBlockIdx; i < pathlen; i++ {
-		if path[i] < '0' || path[i] > '9' {
-			if i == blockQueryBlockIdx {
-				return 0, fmt.Errorf("no block in path")
-			}
-			break
+func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		s, err := parseURL(r.URL.Path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
 		}
-		result = (uint64(10) * result) + uint64(int(path[i])-'0')
+		round, err := strconv.ParseUint(s, 0, 64)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		maybeWriteError(w, gen.WriteDeltas(w, round))
 	}
-	return result, nil
 }
 
-func parseAccount(path string) (string, error) {
-	if !strings.HasPrefix(path, accountsQueryPrefix) {
-		return "", fmt.Errorf("not a accounts query: %s", path)
+func parseURL(path string) (string, error) {
+	i := strings.LastIndex(path, "/")
+	if i == len(path)-1 {
+		return "", fmt.Errorf("invalid request path, %s", path)
 	}
-
-	pathlen := len(path)
-
-	if pathlen == accountsQueryAccountIdx {
-		return "", fmt.Errorf("no address in path")
+	if strings.Contains(path[i+1:], "?") {
+		return strings.Split(path[i+1:], "?")[0], nil
 	}
-
-	return path[accountsQueryAccountIdx:], nil
+	return path[i+1:], nil
 }