tools: new endpoints for block generator

tools/block-generator/generator/generate.go

@@ -17,7 +17,6 @@
 package generator
 
 import (
-	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,6 +25,9 @@ import (
 	"time"
 
 	cconfig "github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 
 	"github.com/algorand/go-algorand/agreement"
@@ -133,6 +135,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
 	gen.genesisHash[31] = 3
 
 	gen.initializeAccounting()
+	gen.initializeLedger()
 
 	for _, val := range getTransactionOptions() {
 		switch val {
@@ -177,7 +180,9 @@ type Generator interface {
 	WriteBlock(output io.Writer, round uint64) error
 	WriteAccount(output io.Writer, accountString string) error
 	WriteStatus(output io.Writer) error
+	WriteDeltas(output io.Writer, round uint64) error
 	Accounts() <-chan basics.Address
+	Stop()
 }
 
 type generator struct {
@@ -224,6 +229,9 @@ type generator struct {
 
 	// Reporting information from transaction type to data
 	reportData Report
+
+	// ledger
+	ledger *ledger.Ledger
 }
 
 type assetData struct {
@@ -410,20 +418,31 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 	if err != nil {
 		return err
 	}
-
+	err = g.ledger.AddBlock(cert.Block, agreement.Certificate{})
+	if err != nil {
+		return err
+	}
+	g.ledger.WaitForCommit(basics.Round(g.round))
 	g.finishRound(numTxnForBlock)
 	return nil
 }
 
-func indexToAccount(i uint64) (addr basics.Address) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	binary.LittleEndian.PutUint64(addr[:], i+1)
-	return
-}
-
-func accountToIndex(a basics.Address) (addr uint64) {
-	// Make sure we don't generate a zero address by adding 1 to i
-	return binary.LittleEndian.Uint64(a[:]) - 1
+// WriteDeltas generates returns the deltas for payset.
+func (g *generator) WriteDeltas(output io.Writer, round uint64) error {
+	delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round))
+	if err != nil {
+		return fmt.Errorf("err getting state delta for round %d, %v", round, err)
+	}
+	// msgp encode deltas
+	data, err := encode(protocol.CodecHandle, delta)
+	if err != nil {
+		return err
+	}
+	_, err = output.Write(data)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // initializeAccounting creates the genesis accounts.
@@ -686,6 +705,35 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S
 	return signTxn(txn), transactions.ApplyData{}, nil
 }
 
+func (g *generator) initializeLedger() {
+	genBal := convertToGenesisBalances(g.balances)
+	// add rewards pool with min balance
+	genBal[g.rewardsPool] = basics.AccountData{
+		MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance},
+	}
+	bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool)
+	block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash)
+	if err != nil {
+		fmt.Printf("error making genesis: %v\n.", err)
+		os.Exit(1)
+	}
+	l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{
+		Block:       block,
+		Accounts:    bal.Balances,
+		GenesisHash: g.genesisHash,
+	}, cconfig.GetDefaultLocal())
+	if err != nil {
+		fmt.Printf("error initializing ledger: %v\n.", err)
+		os.Exit(1)
+	}
+	g.ledger = l
+}
+
+// Stop cleans up allocated resources.
+func (g *generator) Stop() {
+	g.ledger.Close()
+}
+
 func (g *generator) WriteAccount(output io.Writer, accountString string) error {
 	addr, err := basics.UnmarshalChecksumAddress(accountString)
 	if err != nil {

tools/block-generator/generator/generate_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/rpcs"
@@ -31,7 +32,7 @@ func makePrivateGenerator(t *testing.T) *generator {
 	partitiontest.PartitionTest(t)
 	publicGenerator, err := MakeGenerator(GenerationConfig{
 		NumGenesisAccounts:           10,
-		GenesisAccountInitialBalance: 10000000000000000000,
+		GenesisAccountInitialBalance: 1000000000000,
 		PaymentTransactionFraction:   1.0,
 		PaymentNewAccountFraction:    1.0,
 		AssetCreateFraction:          1.0,
@@ -204,13 +205,8 @@ func TestWriteRound(t *testing.T) {
 	var block rpcs.EncodedBlockCert
 	protocol.Decode(data, &block)
 	require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock))
-}
-
-func TestIndexToAccountAndAccountToIndex(t *testing.T) {
-	partitiontest.PartitionTest(t)
-	for i := uint64(0); i < uint64(100000); i++ {
-		acct := indexToAccount(i)
-		result := accountToIndex(acct)
-		require.Equal(t, i, result)
-	}
+	require.NotNil(t, g.ledger)
+	require.Equal(t, basics.Round(1), g.ledger.Latest())
+	_, err := g.ledger.GetStateDeltaForRound(1)
+	require.NoError(t, err)
 }

tools/block-generator/generator/server.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -67,6 +68,8 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B
 	mux.HandleFunc("/genesis", getGenesisHandler(gen))
 	mux.HandleFunc("/report", getReportHandler(gen))
 	mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen))
+	mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {})
+	mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen))
 
 	return &http.Server{
 		Addr:              addr,
@@ -107,68 +110,55 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
 func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// The generator doesn't actually care about the block...
-		round, err := parseRound(r.URL.Path)
+		s, err := parseURL(r.URL.Path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		round, err := strconv.ParseUint(s, 0, 64)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-
 		maybeWriteError(w, gen.WriteBlock(w, round))
 	}
 }
 
 func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// The generator doesn't actually care about the block...
-		account, err := parseAccount(r.URL.Path)
+		account, err := parseURL(r.URL.Path)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-
 		maybeWriteError(w, gen.WriteAccount(w, account))
 	}
 }
 
-const blockQueryPrefix = "/v2/blocks/"
-const blockQueryBlockIdx = len(blockQueryPrefix)
-const accountsQueryPrefix = "/v2/accounts/"
-const accountsQueryAccountIdx = len(accountsQueryPrefix)
-
-func parseRound(path string) (uint64, error) {
-	if !strings.HasPrefix(path, blockQueryPrefix) {
-		return 0, fmt.Errorf("not a blocks query: %s", path)
-	}
-
-	result := uint64(0)
-	pathlen := len(path)
-
-	if pathlen == blockQueryBlockIdx {
-		return 0, fmt.Errorf("no block in path")
-	}
-
-	for i := blockQueryBlockIdx; i < pathlen; i++ {
-		if path[i] < '0' || path[i] > '9' {
-			if i == blockQueryBlockIdx {
-				return 0, fmt.Errorf("no block in path")
-			}
-			break
+func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		s, err := parseURL(r.URL.Path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
 		}
-		result = (uint64(10) * result) + uint64(int(path[i])-'0')
+		round, err := strconv.ParseUint(s, 0, 64)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		maybeWriteError(w, gen.WriteDeltas(w, round))
 	}
-	return result, nil
 }
 
-func parseAccount(path string) (string, error) {
-	if !strings.HasPrefix(path, accountsQueryPrefix) {
-		return "", fmt.Errorf("not a accounts query: %s", path)
+func parseURL(path string) (string, error) {
+	i := strings.LastIndex(path, "/")
+	if i == len(path)-1 {
+		return "", fmt.Errorf("invalid request path, %s", path)
 	}
-
-	pathlen := len(path)
-
-	if pathlen == accountsQueryAccountIdx {
-		return "", fmt.Errorf("no address in path")
+	if strings.Contains(path[i+1:], "?") {
+		return strings.Split(path[i+1:], "?")[0], nil
 	}
-
-	return path[accountsQueryAccountIdx:], nil
+	return path[i+1:], nil
 }

tools/block-generator/generator/server_test.go

@@ -17,13 +17,10 @@
 package generator
 
 import (
-	"fmt"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/algorand/go-algorand/test/partitiontest"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -45,64 +42,24 @@ func TestInitConfigFileNotExist(t *testing.T) {
 	}
 }
 
-func TestParseRound(t *testing.T) {
+func TestParseURL(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	var testcases = []struct {
-		name          string
-		url           string
-		expectedRound uint64
-		err           string
-	}{
-		{
-			name:          "no block",
-			url:           "/v2/blocks/",
-			expectedRound: 0,
-			err:           "no block in path",
-		},
-		{
-			name:          "no block 2",
-			url:           "/v2/blocks/?nothing",
-			expectedRound: 0,
-			err:           "no block in path",
-		},
-		{
-			name:          "invalid prefix",
-			url:           "/v2/wrong/prefix/1",
-			expectedRound: 0,
-			err:           "not a blocks query",
-		},
-		{
-			name:          "normal one digit",
-			url:           fmt.Sprintf("%s1", blockQueryPrefix),
-			expectedRound: 1,
-			err:           "",
-		},
-		{
-			name:          "normal long number",
-			url:           fmt.Sprintf("%s12345678", blockQueryPrefix),
-			expectedRound: 12345678,
-			err:           "",
-		},
-		{
-			name:          "with query parameters",
-			url:           fmt.Sprintf("%s1234?pretty", blockQueryPrefix),
-			expectedRound: 1234,
-			err:           "",
-		},
-	}
+	_, err := parseURL("http://v2/blocks/")
+	require.NotNil(t, err)
+	_, err = parseURL("http://v2/accounts/")
+	require.NotNil(t, err)
+	_, err = parseURL("http://v2/deltas/")
+	require.NotNil(t, err)
 
-	for _, testcase := range testcases {
-		t.Run(testcase.name, func(t *testing.T) {
-			round, err := parseRound(testcase.url)
-			if len(testcase.err) == 0 {
-				msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%d' received error: %v",
-					testcase.url, testcase.expectedRound, err)
-				require.NoError(t, err, msg)
-				assert.Equal(t, testcase.expectedRound, round)
-			} else {
-				require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err))
-				require.True(t, strings.Contains(err.Error(), testcase.err))
-			}
-		})
-	}
+	round, err := parseURL("http://v2/blocks/123")
+	require.Nil(t, err)
+	require.Equal(t, round, "123")
+
+	addr, err := parseURL("http://v2/accounts/AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4")
+	require.Nil(t, err)
+	require.Equal(t, addr, "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4")
+
+	_, err = parseURL("http://v2/deltas/123?Format=msgp")
+	require.Nil(t, err)
+	require.Equal(t, round, "123")
 }