diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 41fad8736e..f1961587bb 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -17,7 +17,6 @@ package generator import ( - "encoding/binary" "encoding/json" "fmt" "io" @@ -26,6 +25,9 @@ import ( "time" cconfig "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/agreement" @@ -133,6 +135,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) { gen.genesisHash[31] = 3 gen.initializeAccounting() + gen.initializeLedger() for _, val := range getTransactionOptions() { switch val { @@ -177,7 +180,9 @@ type Generator interface { WriteBlock(output io.Writer, round uint64) error WriteAccount(output io.Writer, accountString string) error WriteStatus(output io.Writer) error + WriteDeltas(output io.Writer, round uint64) error Accounts() <-chan basics.Address + Stop() } type generator struct { @@ -224,6 +229,9 @@ type generator struct { // Reporting information from transaction type to data reportData Report + + // ledger + ledger *ledger.Ledger } type assetData struct { @@ -410,20 +418,31 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return err } - + err = g.ledger.AddBlock(cert.Block, agreement.Certificate{}) + if err != nil { + return err + } + g.ledger.WaitForCommit(basics.Round(g.round)) g.finishRound(numTxnForBlock) return nil } -func indexToAccount(i uint64) (addr basics.Address) { - // Make sure we don't generate a zero address by adding 1 to i - binary.LittleEndian.PutUint64(addr[:], i+1) - return -} - -func accountToIndex(a basics.Address) (addr uint64) { - // Make sure we don't generate a zero address by adding 1 to i - return binary.LittleEndian.Uint64(a[:]) - 1 +// WriteDeltas generates returns the deltas for payset. +func (g *generator) WriteDeltas(output io.Writer, round uint64) error { + delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round)) + if err != nil { + return fmt.Errorf("err getting state delta for round %d, %v", round, err) + } + // msgp encode deltas + data, err := encode(protocol.CodecHandle, delta) + if err != nil { + return err + } + _, err = output.Write(data) + if err != nil { + return err + } + return nil } // initializeAccounting creates the genesis accounts. @@ -686,6 +705,35 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S return signTxn(txn), transactions.ApplyData{}, nil } +func (g *generator) initializeLedger() { + genBal := convertToGenesisBalances(g.balances) + // add rewards pool with min balance + genBal[g.rewardsPool] = basics.AccountData{ + MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance}, + } + bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool) + block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash) + if err != nil { + fmt.Printf("error making genesis: %v\n.", err) + os.Exit(1) + } + l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{ + Block: block, + Accounts: bal.Balances, + GenesisHash: g.genesisHash, + }, cconfig.GetDefaultLocal()) + if err != nil { + fmt.Printf("error initializing ledger: %v\n.", err) + os.Exit(1) + } + g.ledger = l +} + +// Stop cleans up allocated resources. +func (g *generator) Stop() { + g.ledger.Close() +} + func (g *generator) WriteAccount(output io.Writer, accountString string) error { addr, err := basics.UnmarshalChecksumAddress(accountString) if err != nil { diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 4f25b4d0fb..78ca9e0323 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -18,8 +18,11 @@ package generator import ( "bytes" + "net/http" + "net/http/httptest" "testing" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" @@ -31,7 +34,7 @@ func makePrivateGenerator(t *testing.T) *generator { partitiontest.PartitionTest(t) publicGenerator, err := MakeGenerator(GenerationConfig{ NumGenesisAccounts: 10, - GenesisAccountInitialBalance: 10000000000000000000, + GenesisAccountInitialBalance: 1000000000000, PaymentTransactionFraction: 1.0, PaymentNewAccountFraction: 1.0, AssetCreateFraction: 1.0, @@ -204,13 +207,45 @@ func TestWriteRound(t *testing.T) { var block rpcs.EncodedBlockCert protocol.Decode(data, &block) require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) + require.NotNil(t, g.ledger) + require.Equal(t, basics.Round(1), g.ledger.Latest()) + _, err := g.ledger.GetStateDeltaForRound(1) + require.NoError(t, err) } -func TestIndexToAccountAndAccountToIndex(t *testing.T) { +func TestHandlers(t *testing.T) { partitiontest.PartitionTest(t) - for i := uint64(0); i < uint64(100000); i++ { - acct := indexToAccount(i) - result := accountToIndex(acct) - require.Equal(t, i, result) + g := makePrivateGenerator(t) + handler := getBlockHandler(g) + var testcases = []struct { + name string + url string + err string + }{ + { + name: "no block", + url: "/v2/blocks/?nothing", + err: "invalid request path, /", + }, + { + name: "blocks: round must be numeric", + url: "/v2/blocks/round", + err: `strconv.ParseUint: parsing "round": invalid syntax`, + }, + { + name: "deltas: round must be numeric", + url: "/v2/deltas/round", + err: `strconv.ParseUint: parsing "round": invalid syntax`, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + req := httptest.NewRequest("GET", testcase.url, nil) + w := httptest.NewRecorder() + handler(w, req) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Contains(t, w.Body.String(), testcase.err) + }) } } diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 10adc04728..e0c3314082 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -20,6 +20,7 @@ import ( "fmt" "net/http" "os" + "strconv" "strings" "time" @@ -67,6 +68,8 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B mux.HandleFunc("/genesis", getGenesisHandler(gen)) mux.HandleFunc("/report", getReportHandler(gen)) mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) + mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) + mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen)) return &http.Server{ Addr: addr, @@ -107,12 +110,16 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // The generator doesn't actually care about the block... - round, err := parseRound(r.URL.Path) + s, err := parseURL(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + round, err := strconv.ParseUint(s, 0, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - maybeWriteError(w, gen.WriteBlock(w, round)) } } @@ -120,55 +127,38 @@ func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // The generator doesn't actually care about the block... - account, err := parseAccount(r.URL.Path) + account, err := parseURL(r.URL.Path) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - maybeWriteError(w, gen.WriteAccount(w, account)) } } -const blockQueryPrefix = "/v2/blocks/" -const blockQueryBlockIdx = len(blockQueryPrefix) -const accountsQueryPrefix = "/v2/accounts/" -const accountsQueryAccountIdx = len(accountsQueryPrefix) - -func parseRound(path string) (uint64, error) { - if !strings.HasPrefix(path, blockQueryPrefix) { - return 0, fmt.Errorf("not a blocks query: %s", path) - } - - result := uint64(0) - pathlen := len(path) - - if pathlen == blockQueryBlockIdx { - return 0, fmt.Errorf("no block in path") - } - - for i := blockQueryBlockIdx; i < pathlen; i++ { - if path[i] < '0' || path[i] > '9' { - if i == blockQueryBlockIdx { - return 0, fmt.Errorf("no block in path") - } - break +func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + s, err := parseURL(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } - result = (uint64(10) * result) + uint64(int(path[i])-'0') + round, err := strconv.ParseUint(s, 0, 64) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + maybeWriteError(w, gen.WriteDeltas(w, round)) } - return result, nil } -func parseAccount(path string) (string, error) { - if !strings.HasPrefix(path, accountsQueryPrefix) { - return "", fmt.Errorf("not a accounts query: %s", path) +func parseURL(path string) (string, error) { + i := strings.LastIndex(path, "/") + if i == len(path)-1 { + return "", fmt.Errorf("invalid request path, %s", path) } - - pathlen := len(path) - - if pathlen == accountsQueryAccountIdx { - return "", fmt.Errorf("no address in path") + if strings.Contains(path[i+1:], "?") { + return strings.Split(path[i+1:], "?")[0], nil } - - return path[accountsQueryAccountIdx:], nil + return path[i+1:], nil } diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 4d2863aeff..6e21d77586 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -45,57 +45,78 @@ func TestInitConfigFileNotExist(t *testing.T) { } } -func TestParseRound(t *testing.T) { +func TestParseURL(t *testing.T) { partitiontest.PartitionTest(t) + const blockQueryPrefix = "http://v2/blocks/" + const accountQueryPrefix = "http://v2/accounts/" + const deltaQueryPrefix = "http://v2/deltas/" var testcases = []struct { name string url string - expectedRound uint64 + expectedRound string err string }{ { name: "no block", url: "/v2/blocks/", - expectedRound: 0, - err: "no block in path", - }, - { - name: "no block 2", - url: "/v2/blocks/?nothing", - expectedRound: 0, - err: "no block in path", - }, - { - name: "invalid prefix", - url: "/v2/wrong/prefix/1", - expectedRound: 0, - err: "not a blocks query", + expectedRound: "", + err: "invalid request path, /v2/blocks/", }, { name: "normal one digit", url: fmt.Sprintf("%s1", blockQueryPrefix), - expectedRound: 1, + expectedRound: "1", err: "", }, { name: "normal long number", url: fmt.Sprintf("%s12345678", blockQueryPrefix), - expectedRound: 12345678, + expectedRound: "12345678", + err: "", + }, + { + name: "with query parameters", + url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), + expectedRound: "1234", err: "", }, { name: "with query parameters", url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), - expectedRound: 1234, + expectedRound: "1234", + err: "", + }, + { + name: "no deltas", + url: "/v2/deltas/", + expectedRound: "", + err: "invalid request path, /v2/deltas/", + }, + { + name: "deltas", + url: fmt.Sprintf("%s123?Format=msgp", deltaQueryPrefix), + expectedRound: "123", + err: "", + }, + { + name: "no account", + url: "/v2/accounts/", + expectedRound: "", + err: "invalid request path, /v2/accounts/", + }, + { + name: "accounts", + url: fmt.Sprintf("%sAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", accountQueryPrefix), + expectedRound: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", err: "", }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - round, err := parseRound(testcase.url) + round, err := parseURL(testcase.url) if len(testcase.err) == 0 { - msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%d' received error: %v", + msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%s' received error: %v", testcase.url, testcase.expectedRound, err) require.NoError(t, err, msg) assert.Equal(t, testcase.expectedRound, round) diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index 7c7a3980ce..59ff7f2e06 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -17,8 +17,12 @@ package generator import ( + "encoding/binary" "fmt" "math/rand" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-codec/codec" ) func weightedSelection(weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) { @@ -43,3 +47,35 @@ func weightedSelectionInternal(selectionNumber float32, weights []float32, optio selection = defaultOption return } + +func indexToAccount(i uint64) (addr basics.Address) { + // Make sure we don't generate a zero address by adding 1 to i + binary.LittleEndian.PutUint64(addr[:], i+1) + return +} + +func accountToIndex(a basics.Address) (addr uint64) { + // Make sure we don't generate a zero address by adding 1 to i + return binary.LittleEndian.Uint64(a[:]) - 1 +} + +func convertToGenesisBalances(balances []uint64) map[basics.Address]basics.AccountData { + genesisBalances := make(map[basics.Address]basics.AccountData) + for i, balance := range balances { + genesisBalances[indexToAccount(uint64(i))] = basics.AccountData{ + MicroAlgos: basics.MicroAlgos{Raw: balance}, + } + } + return genesisBalances +} + +func encode(handle codec.Handle, obj interface{}) ([]byte, error) { + var output []byte + enc := codec.NewEncoderBytes(&output, handle) + + err := enc.Encode(obj) + if err != nil { + return nil, fmt.Errorf("failed to encode object: %v", err) + } + return output, nil +} diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 7fc289d695..fb69a43fee 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -109,3 +109,24 @@ func TestWeightedSelectionOutOfRange(t *testing.T) { } assert.Fail(t, "Expected an out of range error by this point.") } + +func TestConvertToGenesisBalance(t *testing.T) { + partitiontest.PartitionTest(t) + balance := []uint64{100, 200, 300} + genesisBalances := convertToGenesisBalances(balance) + require.Equal(t, 3, len(genesisBalances)) + for i, bal := range balance { + require.Equal(t, bal, genesisBalances[indexToAccount(uint64(i))].MicroAlgos.Raw) + } +} + +func TestIndexToAccountAndAccountToIndex(t *testing.T) { + partitiontest.PartitionTest(t) + account := indexToAccount(1) + fmt.Printf("account: %v\n", account) + for i := uint64(0); i < uint64(100000); i++ { + acct := indexToAccount(i) + result := accountToIndex(acct) + require.Equal(t, i, result) + } +} diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 7d933c557b..9604212da1 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -341,6 +341,8 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha }() return func() error { + // stop generator + defer generator.Stop() // Shutdown blocks until the server has stopped. if err := server.Shutdown(context.Background()); err != nil { return fmt.Errorf("failed during generator graceful shutdown: %w", err)