From 49da1fcb202dd547a3b31b489afed7ee2987584b Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 30 Mar 2023 13:14:22 -0400 Subject: [PATCH 01/16] block generator builds --- tools/block-generator/README.md | 115 +++ tools/block-generator/core/commands.go | 19 + tools/block-generator/generator/daemon.go | 33 + tools/block-generator/generator/generate.go | 749 ++++++++++++++++++ .../generator/generate_test.go | 186 +++++ .../generator/make_transactions.go | 81 ++ tools/block-generator/generator/server.go | 153 ++++ .../block-generator/generator/server_test.go | 88 ++ tools/block-generator/generator/utils.go | 29 + tools/block-generator/generator/utils_test.go | 90 +++ tools/block-generator/main.go | 7 + tools/block-generator/metrics/metrics.go | 26 + tools/block-generator/run_generator.sh | 56 ++ tools/block-generator/run_postgres.sh | 49 ++ tools/block-generator/run_runner.sh | 54 ++ tools/block-generator/run_tests.sh | 98 +++ .../runner/metrics_collector.go | 72 ++ tools/block-generator/runner/run.go | 331 ++++++++ tools/block-generator/runner/runner.go | 45 ++ .../scenarios/config.asset.close.yml | 16 + .../scenarios/config.asset.destroy.yml | 16 + .../scenarios/config.asset.xfer.yml | 15 + .../scenarios/config.mixed.jumbo.yml | 19 + .../scenarios/config.mixed.yml | 19 + .../scenarios/config.payment.full.yml | 14 + .../scenarios/config.payment.jumbo.yml | 14 + .../scenarios/config.payment.small.yml | 14 + tools/block-generator/test_config.yml | 23 + tools/block-generator/util/util.go | 16 + 29 files changed, 2447 insertions(+) create mode 100644 tools/block-generator/README.md create mode 100644 tools/block-generator/core/commands.go create mode 100644 tools/block-generator/generator/daemon.go create mode 100644 tools/block-generator/generator/generate.go create mode 100644 tools/block-generator/generator/generate_test.go create mode 100644 tools/block-generator/generator/make_transactions.go create mode 100644 tools/block-generator/generator/server.go create mode 100644 tools/block-generator/generator/server_test.go create mode 100644 tools/block-generator/generator/utils.go create mode 100644 tools/block-generator/generator/utils_test.go create mode 100644 tools/block-generator/main.go create mode 100644 tools/block-generator/metrics/metrics.go create mode 100755 tools/block-generator/run_generator.sh create mode 100755 tools/block-generator/run_postgres.sh create mode 100755 tools/block-generator/run_runner.sh create mode 100755 tools/block-generator/run_tests.sh create mode 100644 tools/block-generator/runner/metrics_collector.go create mode 100644 tools/block-generator/runner/run.go create mode 100644 tools/block-generator/runner/runner.go create mode 100644 tools/block-generator/scenarios/config.asset.close.yml create mode 100644 tools/block-generator/scenarios/config.asset.destroy.yml create mode 100644 tools/block-generator/scenarios/config.asset.xfer.yml create mode 100644 tools/block-generator/scenarios/config.mixed.jumbo.yml create mode 100644 tools/block-generator/scenarios/config.mixed.yml create mode 100644 tools/block-generator/scenarios/config.payment.full.yml create mode 100644 tools/block-generator/scenarios/config.payment.jumbo.yml create mode 100644 tools/block-generator/scenarios/config.payment.small.yml create mode 100644 tools/block-generator/test_config.yml create mode 100644 tools/block-generator/util/util.go diff --git a/tools/block-generator/README.md b/tools/block-generator/README.md new file mode 100644 index 0000000000..5caff65225 --- /dev/null +++ b/tools/block-generator/README.md @@ -0,0 +1,115 @@ +# Block Generator + +This tool is used for testing Indexer import performance. It does this by generating synthetic blocks which are sent by mocking the Algod REST API endpoints that Indexer uses. + +## Scenario Configuration + +Block generator uses a YAML config file to describe the composition of each randomly generated block. There are three levels of configuration: +1. Setup +2. Transaction type distribution +3. Transaction type specific configuration + +At the time of writing, the block generator supports **payment** and **asset** transactions. The settings are hopefully, more or less, obvious. Distributions are specified as fractions of one, and the sum of all options must add up to one. + +Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, in this case the block size is specified as 19999, or four times larger than the current block size limit: +``` +name: "Mixed (jumbo)" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 19999 + +# transaction distribution +tx_pay_fraction: 0.3 +tx_asset_fraction: 0.7 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.1 +asset_close_fraction: 0.05 +asset_xfer_fraction: 0.849 +asset_delete_fraction: 0 +``` + +## Modes + +The block generator can run in one of two modes, a standalone **daemon**, or a test suite **runner** + +### daemon + +In standalone mode, a block-generator process starts and exposes the mock algod endpoints for **/genesis** and **/v2/blocks/{block}**. If you choose to query them manually, it only supports fetching blocks sequentially. This is due to the fact that it generates a pseudorandom stream of transactions and after each random transaction the state increments to the next. + +Here is the help output: +```bash +~$ ./block-generator daemon -h +Start the generator daemon in standalone mode. + +Usage: + block-generator daemon [flags] + +Flags: + -c, --config string Specify the block configuration yaml file. + -h, --help help for daemon + -p, --port uint Port to start the server at. (default 4010) +``` + +### runner + +For our usage, we want to run the same set of tests consistently across many scenarios and with many different releases. The runner mode automates this process by starting the **daemon** with many different configurations, managing a postgres database, and running a separate indexer process configured to use them. + +The results of the testing are written to the directory specified by the **--report-directory** option, and include many different metrics. In addition to the report, the indexer log is written to this directory. The files are named according to the scenario file, and end in "report" or "log". + +Here is an example report from running with a test duration of "1h": +``` +test_duration_seconds:3600 +test_duration_actual_seconds:3600.056457 +transaction_pay_total:30024226 +transaction_pay_create_total:614242 +early_average_import_time_sec:2.13 +early_cumulative_import_time_sec:1083.26 +early_average_imported_tx_per_block:19999.00 +early_cumulative_imported_tx_per_block:10179491 +early_average_block_upload_time_sec:NaN +early_cumulative_block_upload_time_sec:0.00 +early_average_postgres_eval_time_sec:0.33 +early_cumulative_postgres_eval_time_sec:167.41 +early_imported_round:509 +early_overall_transactions_per_second:9397.09 +early_uptime_seconds:3600.06 +final_average_import_time_sec:2.35 +final_cumulative_import_time_sec:3602.62 +final_average_imported_tx_per_block:19999.00 +final_cumulative_imported_tx_per_block:30598470 +final_average_block_upload_time_sec:NaN +final_cumulative_block_upload_time_sec:0.00 +final_average_postgres_eval_time_sec:0.33 +final_cumulative_postgres_eval_time_sec:507.38 +final_imported_round:1530 +final_overall_transactions_per_second:8493.40 +final_uptime_seconds:3600.06 +``` + +Here is the help output: +```bash +~$ ./block-generator runner -h +Run test suite and collect results. + +Usage: + block-generator runner [flags] + +Flags: + --cpuprofile string Path where Indexer writes its CPU profile. + -h, --help help for runner + -i, --indexer-binary string Path to indexer binary. + -p, --indexer-port uint Port to start the server at. This is useful if you have a prometheus server for collecting additional data. (default 4010) + -l, --log-level string LogLevel to use when starting Indexer. [error, warn, info, debug, trace] (default "error") + -c, --postgres-connection-string string Postgres connection string. + -r, --report-directory string Location to place test reports. + --reset If set any existing report directory will be deleted before running tests. + -s, --scenario string Directory containing scenarios, or specific scenario file. + -d, --test-duration duration Duration to use for each scenario. (default 5m0s) + --validate If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure. +``` diff --git a/tools/block-generator/core/commands.go b/tools/block-generator/core/commands.go new file mode 100644 index 0000000000..8fb969c294 --- /dev/null +++ b/tools/block-generator/core/commands.go @@ -0,0 +1,19 @@ +package core + +import ( + "github.com/algorand/go-algorand/tools/block-generator/generator" + "github.com/algorand/go-algorand/tools/block-generator/runner" + "github.com/spf13/cobra" +) + +// BlockGenerator related cobra commands, ready to be executed or included as subcommands. +var BlockGenerator *cobra.Command + +func init() { + BlockGenerator = &cobra.Command{ + Use: `block-generator`, + Short: `Block generator testing tools.`, + } + BlockGenerator.AddCommand(runner.RunnerCmd) + BlockGenerator.AddCommand(generator.DaemonCmd) +} diff --git a/tools/block-generator/generator/daemon.go b/tools/block-generator/generator/daemon.go new file mode 100644 index 0000000000..ac60a40edf --- /dev/null +++ b/tools/block-generator/generator/daemon.go @@ -0,0 +1,33 @@ +package generator + +import ( + "fmt" + "math/rand" + + "github.com/spf13/cobra" +) + +// DaemonCmd starts a block generator daemon. +var DaemonCmd *cobra.Command + +func init() { + rand.Seed(12345) + + var configFile string + var port uint64 + + DaemonCmd = &cobra.Command{ + Use: "daemon", + Short: "Start the generator daemon in standalone mode.", + Run: func(cmd *cobra.Command, args []string) { + addr := fmt.Sprintf(":%d", port) + srv, _ := MakeServer(configFile, addr) + srv.ListenAndServe() + }, + } + + DaemonCmd.Flags().StringVarP(&configFile, "config", "c", "", "Specify the block configuration yaml file.") + DaemonCmd.Flags().Uint64VarP(&port, "port", "p", 4010, "Port to start the server at.") + + DaemonCmd.MarkFlagRequired("config") +} diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go new file mode 100644 index 0000000000..88cf102054 --- /dev/null +++ b/tools/block-generator/generator/generate.go @@ -0,0 +1,749 @@ +package generator + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "time" + + cconfig "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/protocol" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated/model" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/committee" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/rpcs" +) + +// TxTypeID is the transaction type. +type TxTypeID string + +const ( + genesis TxTypeID = "genesis" + + // Payment Tx IDs + paymentTx TxTypeID = "pay" + paymentAcctCreateTx TxTypeID = "pay_create" + assetTx TxTypeID = "asset" + //keyRegistrationTx TxTypeID = "keyreg" + //applicationCallTx TxTypeID = "appl" + + // Asset Tx IDs + assetCreate TxTypeID = "asset_create" + assetOptin TxTypeID = "asset_optin" + assetXfer TxTypeID = "asset_xfer" + assetClose TxTypeID = "asset_close" + assetDestroy TxTypeID = "asset_destroy" + + assetTotal = uint64(100000000000000000) + + consensusTimeMilli int64 = 4500 +) + +// GenerationConfig defines the tunable parameters for block generation. +type GenerationConfig struct { + Name string `mapstructure:"name"` + NumGenesisAccounts uint64 `mapstructure:"genesis_accounts"` + GenesisAccountInitialBalance uint64 `mapstructure:"genesis_account_balance"` + + // Block generation + TxnPerBlock uint64 `mapstructure:"tx_per_block"` + + // TX Distribution + PaymentTransactionFraction float32 `mapstructure:"tx_pay_fraction"` + AssetTransactionFraction float32 `mapstructure:"tx_asset_fraction"` + + // Payment configuration + PaymentNewAccountFraction float32 `mapstructure:"pay_acct_create_fraction"` + PaymentFraction float32 `mapstructure:"pay_xfer_fraction"` + + // Asset configuration + AssetCreateFraction float32 `mapstructure:"asset_create_fraction"` + AssetDestroyFraction float32 `mapstructure:"asset_destroy_fraction"` + AssetOptinFraction float32 `mapstructure:"asset_optin_fraction"` + AssetCloseFraction float32 `mapstructure:"asset_close_fraction"` + AssetXferFraction float32 `mapstructure:"asset_xfer_fraction"` +} + +func sumIsCloseToOne(numbers ...float32) bool { + var sum float32 + for _, num := range numbers { + sum += num + } + return sum > 0.99 && sum < 1.01 +} + +// MakeGenerator initializes the Generator object. +func MakeGenerator(config GenerationConfig) (Generator, error) { + if !sumIsCloseToOne(config.PaymentTransactionFraction, config.AssetTransactionFraction) { + return nil, fmt.Errorf("transaction distribution ratios should equal 1") + } + + if !sumIsCloseToOne(config.PaymentNewAccountFraction, config.PaymentFraction) { + return nil, fmt.Errorf("payment configuration ratios should equal 1") + } + + if !sumIsCloseToOne(config.AssetCreateFraction, config.AssetDestroyFraction, config.AssetOptinFraction, config.AssetCloseFraction, config.AssetXferFraction) { + return nil, fmt.Errorf("asset configuration ratios should equal 1") + } + + var proto protocol.ConsensusVersion = "future" + gen := &generator{ + config: config, + protocol: proto, + params: cconfig.Consensus[proto], + genesisHash: [32]byte{}, + genesisID: "blockgen-test", + prevBlockHash: "", + round: 1, + txnCounter: 0, + timestamp: 0, + rewardsLevel: 0, + rewardsResidue: 0, + rewardsRate: 0, + rewardsRecalculationRound: 0, + reportData: make(map[TxTypeID]TxData), + } + + gen.feeSink[31] = 1 + gen.rewardsPool[31] = 2 + gen.genesisHash[31] = 3 + + gen.initializeAccounting() + + for _, val := range getTransactionOptions() { + switch val { + case paymentTx: + gen.transactionWeights = append(gen.transactionWeights, config.PaymentTransactionFraction) + case assetTx: + gen.transactionWeights = append(gen.transactionWeights, config.AssetTransactionFraction) + } + } + + for _, val := range getPaymentTxOptions() { + switch val { + case paymentTx: + gen.payTxWeights = append(gen.payTxWeights, config.PaymentFraction) + case paymentAcctCreateTx: + gen.payTxWeights = append(gen.payTxWeights, config.PaymentNewAccountFraction) + } + } + + for _, val := range getAssetTxOptions() { + switch val { + case assetCreate: + gen.assetTxWeights = append(gen.assetTxWeights, config.AssetCreateFraction) + case assetDestroy: + gen.assetTxWeights = append(gen.assetTxWeights, config.AssetDestroyFraction) + case assetOptin: + gen.assetTxWeights = append(gen.assetTxWeights, config.AssetOptinFraction) + case assetXfer: + gen.assetTxWeights = append(gen.assetTxWeights, config.AssetXferFraction) + case assetClose: + gen.assetTxWeights = append(gen.assetTxWeights, config.AssetCloseFraction) + } + } + + return gen, nil +} + +// Generator is the interface needed to generate blocks. +type Generator interface { + WriteReport(output io.Writer) error + WriteGenesis(output io.Writer) error + WriteBlock(output io.Writer, round uint64) error + WriteAccount(output io.Writer, accountString string) error + WriteStatus(output io.Writer) error + Accounts() <-chan basics.Address +} + +type generator struct { + config GenerationConfig + + // payment transaction metadata + numPayments uint64 + + // Number of algorand accounts + numAccounts uint64 + + // Block stuff + round uint64 + txnCounter uint64 + prevBlockHash string + timestamp int64 + protocol protocol.ConsensusVersion + params cconfig.ConsensusParams + genesisID string + genesisHash crypto.Digest + + // Rewards stuff + feeSink basics.Address + rewardsPool basics.Address + rewardsLevel uint64 + rewardsResidue uint64 + rewardsRate uint64 + rewardsRecalculationRound uint64 + + // balances for all accounts. To avoid crypto and reduce storage, accounts are faked. + // The account is based on the index into the balances array. + balances []uint64 + + // assets is a minimal representation of the asset holdings, it doesn't + // include the frozen state. + assets []*assetData + // pendingAssets is used to hold newly created assets so that they are not used before + // being created. + pendingAssets []*assetData + + transactionWeights []float32 + payTxWeights []float32 + assetTxWeights []float32 + + // Reporting information from transaction type to data + reportData Report +} + +type assetData struct { + assetID uint64 + creator uint64 + name string + // Holding at index 0 is the creator. + holdings []*assetHolding + // Set of holders in the holdings array for easy reference. + holders map[uint64]*assetHolding +} + +type assetHolding struct { + acctIndex uint64 + balance uint64 +} + +// Report is the generation report. +type Report map[TxTypeID]TxData + +// TxData is the generator report data. +type TxData struct { + GenerationTime time.Duration `json:"generation_time_milli"` + GenerationCount uint64 `json:"num_generated"` +} + +func track(id TxTypeID) (TxTypeID, time.Time) { + return id, time.Now() +} +func (g *generator) recordData(id TxTypeID, start time.Time) { + data := g.reportData[id] + data.GenerationCount++ + data.GenerationTime += time.Since(start) + g.reportData[id] = data +} + +func (g *generator) WriteReport(output io.Writer) error { + return json.NewEncoder(output).Encode(g.reportData) +} + +func (g *generator) WriteStatus(output io.Writer) error { + response := model.NodeStatusResponse{ + LastRound: g.round, + } + return json.NewEncoder(output).Encode(response) +} + +func (g *generator) WriteGenesis(output io.Writer) error { + defer g.recordData(track(genesis)) + var allocations []bookkeeping.GenesisAllocation + + for i := uint64(0); i < g.config.NumGenesisAccounts; i++ { + addr := indexToAccount(i) + allocations = append(allocations, bookkeeping.GenesisAllocation{ + Address: addr.String(), + State: basics.AccountData{ + MicroAlgos: basics.MicroAlgos{Raw: g.config.GenesisAccountInitialBalance}, + }, + }) + } + // Also add the rewards pool account with minimum balance. Without it, the evaluator + // crashes. + allocations = append(allocations, bookkeeping.GenesisAllocation{ + Address: g.rewardsPool.String(), + Comment: "RewardsPool", + State: basics.AccountData{ + MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance}, + Status: basics.NotParticipating, + }, + }) + + gen := bookkeeping.Genesis{ + SchemaID: "v1", + Network: "generated-network", + Proto: g.protocol, + Allocation: allocations, + RewardsPool: g.rewardsPool.String(), + FeeSink: g.feeSink.String(), + Timestamp: g.timestamp, + } + return json.NewEncoder(output).Encode(gen) +} + +func getTransactionOptions() []interface{} { + return []interface{}{paymentTx, assetTx} +} + +func (g *generator) generateTransaction(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { + selection, err := weightedSelection(g.transactionWeights, getTransactionOptions(), paymentTx) + if err != nil { + return transactions.SignedTxn{}, transactions.ApplyData{}, err + } + + switch selection { + case paymentTx: + return g.generatePaymentTxn(round, intra) + case assetTx: + return g.generateAssetTxn(round, intra) + default: + return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("no generator available for %s", selection) + } +} + +func (g *generator) txnForRound(round uint64) uint64 { + // There are no transactions in the 0th round + if round == 0 { + return 0 + } + return g.config.TxnPerBlock +} + +// finishRound tells the generator it can apply any pending state. +func (g *generator) finishRound(txnCount uint64) { + g.txnCounter += txnCount + + g.timestamp += consensusTimeMilli + g.round++ + + // Apply pending assets... + g.assets = append(g.assets, g.pendingAssets...) + g.pendingAssets = nil +} + +// WriteBlock generates a block full of new transactions and writes it to the writer. +func (g *generator) WriteBlock(output io.Writer, round uint64) error { + if round != g.round { + fmt.Printf("Generator only supports sequential block access. Expected %d but received request for %d.", g.round, round) + } + + numTxnForBlock := g.txnForRound(round) + + header := bookkeeping.BlockHeader{ + Round: basics.Round(g.round), + Branch: bookkeeping.BlockHash{}, + Seed: committee.Seed{}, + TxnCommitments: bookkeeping.TxnCommitments{NativeSha512_256Commitment: crypto.Digest{}}, + TimeStamp: g.timestamp, + GenesisID: g.genesisID, + GenesisHash: g.genesisHash, + RewardsState: bookkeeping.RewardsState{ + FeeSink: g.feeSink, + RewardsPool: g.rewardsPool, + RewardsLevel: 0, + RewardsRate: 0, + RewardsResidue: 0, + RewardsRecalculationRound: 0, + }, + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: g.protocol, + }, + UpgradeVote: bookkeeping.UpgradeVote{}, + TxnCounter: g.txnCounter + numTxnForBlock, + StateProofTracking: nil, + } + + // Generate the transactions + transactions := make([]transactions.SignedTxnInBlock, 0, numTxnForBlock) + + for i := uint64(0); i < numTxnForBlock; i++ { + txn, ad, err := g.generateTransaction(g.round, i) + if err != nil { + panic(fmt.Sprintf("failed to generate transaction: %v\n", err)) + } + stib, err := header.EncodeSignedTxn(txn, ad) + if err != nil { + panic(fmt.Sprintf("failed to encode transaction: %v\n", err)) + } + transactions = append(transactions, stib) + } + + if numTxnForBlock != uint64(len(transactions)) { + panic("Unexpected number of transactions.") + } + + cert := rpcs.EncodedBlockCert{ + Block: bookkeeping.Block{ + BlockHeader: header, + Payset: transactions, + }, + Certificate: agreement.Certificate{}, + } + + err := json.NewEncoder(output).Encode(cert) + if err != nil { + return err + } + + g.finishRound(numTxnForBlock) + return nil +} + +func indexToAccount(i uint64) (addr basics.Address) { + // Make sure we don't generate a zero address by adding 1 to i + binary.LittleEndian.PutUint64(addr[:], i+1) + return +} + +func accountToIndex(a basics.Address) (addr uint64) { + // Make sure we don't generate a zero address by adding 1 to i + return binary.LittleEndian.Uint64(a[:]) - 1 +} + +// initializeAccounting creates the genesis accounts. +func (g *generator) initializeAccounting() { + if g.config.NumGenesisAccounts == 0 { + panic("Number of genesis accounts must be > 0.") + } + + g.numPayments = 0 + g.numAccounts = g.config.NumGenesisAccounts + for i := uint64(0); i < g.config.NumGenesisAccounts; i++ { + g.balances = append(g.balances, g.config.GenesisAccountInitialBalance) + } +} + +func signTxn(txn transactions.Transaction) transactions.SignedTxn { + stxn := transactions.SignedTxn{ + Sig: crypto.Signature{}, + Msig: crypto.MultisigSig{}, + Lsig: transactions.LogicSig{}, + Txn: txn, + AuthAddr: basics.Address{}, + } + + // TODO: Would it be useful to generate a random signature? + stxn.Sig[32] = 50 + + return stxn +} + +func getPaymentTxOptions() []interface{} { + return []interface{}{paymentTx, paymentAcctCreateTx} +} + +// generatePaymentTxn creates a new payment transaction. The sender is always a genesis account, the receiver is random, +// or a new account. +func (g *generator) generatePaymentTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { + selection, err := weightedSelection(g.payTxWeights, getPaymentTxOptions(), paymentTx) + if err != nil { + return transactions.SignedTxn{}, transactions.ApplyData{}, err + } + return g.generatePaymentTxnInternal(selection.(TxTypeID), round, intra) +} + +func (g *generator) generatePaymentTxnInternal(selection TxTypeID, round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { + defer g.recordData(track(selection)) + minBal := g.params.MinBalance + + // default amount + amount := uint64(1) + + // Select a receiver + var receiveIndex uint64 + switch selection { + case paymentTx: + receiveIndex = rand.Uint64() % g.numAccounts + case paymentAcctCreateTx: + // give new accounts get extra algos for sending other transactions + amount = minBal * 100 + g.balances = append(g.balances, 0) + receiveIndex = g.numAccounts + g.numAccounts++ + } + total := amount + g.params.MinTxnFee + + // Select a sender from genesis account + sendIndex := g.numPayments % g.config.NumGenesisAccounts + if g.balances[sendIndex] < (total + minBal) { + fmt.Printf("\n\ngeneratePaymentTxnInternal(): the sender account does not have enough algos for the transfer. idx %d, payment number %d\n\n", sendIndex, g.numPayments) + os.Exit(1) + } + + sender := indexToAccount(sendIndex) + receiver := indexToAccount(receiveIndex) + + g.balances[sendIndex] -= total + g.balances[receiveIndex] += amount + + g.numPayments++ + + txn := g.makePaymentTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}) + return signTxn(txn), transactions.ApplyData{}, nil +} + +func getAssetTxOptions() []interface{} { + return []interface{}{assetCreate, assetDestroy, assetOptin, assetXfer, assetClose} +} + +func (g *generator) generateAssetTxnInternal(txType TxTypeID, round uint64, intra uint64) (actual TxTypeID, txn transactions.Transaction) { + return g.generateAssetTxnInternalHint(txType, round, intra, 0, nil) +} + +func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, intra uint64, hintIndex uint64, hint *assetData) (actual TxTypeID, txn transactions.Transaction) { + actual = txType + // If there are no assets the next operation needs to be a create. + if len(g.assets) == 0 { + actual = assetCreate + } + + numAssets := uint64(len(g.assets)) + var senderIndex uint64 + + if actual == assetCreate { + numAssets = uint64(len(g.assets)) + uint64(len(g.pendingAssets)) + senderIndex = numAssets % g.config.NumGenesisAccounts + senderAcct := indexToAccount(senderIndex) + + total := assetTotal + assetID := g.txnCounter + intra + 1 + assetName := fmt.Sprintf("asset #%d", assetID) + txn = g.makeAssetCreateTxn(g.makeTxnHeader(senderAcct, round, intra), total, false, assetName) + + // Compute asset ID and initialize holdings + holding := assetHolding{ + acctIndex: senderIndex, + balance: total, + } + a := assetData{ + name: assetName, + assetID: assetID, + creator: senderIndex, + holdings: []*assetHolding{&holding}, + holders: map[uint64]*assetHolding{senderIndex: &holding}, + } + + g.pendingAssets = append(g.pendingAssets, &a) + } else { + assetIndex := rand.Uint64() % numAssets + asset := g.assets[assetIndex] + if hint != nil { + assetIndex = hintIndex + asset = hint + } + + switch actual { + case assetDestroy: + // delete asset + + // If the creator doesn't have all of them, close instead + if asset.holdings[0].balance != assetTotal { + return g.generateAssetTxnInternalHint(assetClose, round, intra, assetIndex, asset) + } + + senderIndex = asset.creator + creator := indexToAccount(senderIndex) + txn = g.makeAssetDestroyTxn(g.makeTxnHeader(creator, round, intra), asset.assetID) + + // Remove asset by moving the last element to the deleted index then trimming the slice. + g.assets[assetIndex] = g.assets[numAssets-1] + g.assets = g.assets[:numAssets-1] + case assetOptin: + // select a random account from asset to optin + + // If every account holds the asset, close instead of optin + if uint64(len(asset.holdings)) == g.numAccounts { + return g.generateAssetTxnInternalHint(assetClose, round, intra, assetIndex, asset) + } + + // look for an account that does not hold the asset + exists := true + for exists { + senderIndex = rand.Uint64() % g.numAccounts + exists = asset.holders[senderIndex] != nil + } + account := indexToAccount(senderIndex) + txn = g.makeAssetAcceptanceTxn(g.makeTxnHeader(account, round, intra), asset.assetID) + + holding := assetHolding{ + acctIndex: senderIndex, + balance: 0, + } + asset.holdings = append(asset.holdings, &holding) + asset.holders[senderIndex] = &holding + case assetXfer: + // send from creator (holder[0]) to another random holder (same address is valid) + + // If there aren't enough assets to close one, optin an account instead + if len(asset.holdings) == 1 { + return g.generateAssetTxnInternalHint(assetOptin, round, intra, assetIndex, asset) + } + + senderIndex = asset.holdings[0].acctIndex + sender := indexToAccount(senderIndex) + + receiverArrayIndex := (rand.Uint64() % (uint64(len(asset.holdings)) - uint64(1))) + uint64(1) + receiver := indexToAccount(asset.holdings[receiverArrayIndex].acctIndex) + + amount := uint64(10) + + txn = g.makeAssetTransferTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}, asset.assetID) + + if asset.holdings[0].balance < amount { + fmt.Printf("\n\ncreator doesn't have enough funds for asset %d\n\n", asset.assetID) + os.Exit(1) + } + if g.balances[asset.holdings[0].acctIndex] < g.params.MinTxnFee { + fmt.Printf("\n\ncreator doesn't have enough funds for transaction %d\n\n", asset.assetID) + os.Exit(1) + } + + asset.holdings[0].balance -= amount + asset.holdings[receiverArrayIndex].balance += amount + case assetClose: + // select a holder of a random asset to close out + // If there aren't enough assets to close one, optin an account instead + if len(asset.holdings) == 1 { + return g.generateAssetTxnInternalHint( + assetOptin, round, intra, assetIndex, asset) + } + + numHoldings := uint64(len(asset.holdings)) + closeIndex := (rand.Uint64() % (numHoldings - 1)) + uint64(1) + senderIndex = asset.holdings[closeIndex].acctIndex + sender := indexToAccount(senderIndex) + + closeToAcctIndex := asset.holdings[0].acctIndex + closeToAcct := indexToAccount(closeToAcctIndex) + + txn = g.makeAssetTransferTxn( + g.makeTxnHeader(sender, round, intra), closeToAcct, 0, closeToAcct, asset.assetID) + + asset.holdings[0].balance += asset.holdings[closeIndex].balance + + // Remove asset by moving the last element to the deleted index then trimming the slice. + asset.holdings[closeIndex] = asset.holdings[numHoldings-1] + asset.holdings = asset.holdings[:numHoldings-1] + delete(asset.holders, senderIndex) + default: + } + } + + if indexToAccount(senderIndex) != txn.Sender { + fmt.Printf("failed to properly set sender index.") + os.Exit(1) + } + + if g.balances[senderIndex] < txn.Fee.ToUint64() { + fmt.Printf("\n\nthe sender account does not have enough algos for the transfer. idx %d, asset transaction type %v, num %d\n\n", senderIndex, actual, g.reportData[actual].GenerationCount) + os.Exit(1) + } + g.balances[senderIndex] -= txn.Fee.ToUint64() + return +} + +func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { + start := time.Now() + selection, err := weightedSelection(g.assetTxWeights, getAssetTxOptions(), assetXfer) + if err != nil { + return transactions.SignedTxn{}, transactions.ApplyData{}, err + } + + actual, txn := g.generateAssetTxnInternal(selection.(TxTypeID), round, intra) + defer g.recordData(actual, start) + + if txn.Type == "" { + fmt.Println("Empty asset transaction.") + os.Exit(1) + } + + return signTxn(txn), transactions.ApplyData{}, nil +} + +func (g *generator) WriteAccount(output io.Writer, accountString string) error { + addr, err := basics.UnmarshalChecksumAddress(accountString) + if err != nil { + return fmt.Errorf("failed to unmarshal address: %w", err) + } + + idx := accountToIndex(addr) + + // Asset Holdings + assets := make([]model.AssetHolding, 0) + createdAssets := make([]model.Asset, 0) + for _, a := range g.assets { + // holdings + if holding := a.holders[idx]; holding != nil { + assets = append(assets, model.AssetHolding{ + Amount: holding.balance, + AssetID: a.assetID, + IsFrozen: false, + }) + } + // creator + if len(a.holdings) > 0 && a.holdings[0].acctIndex == idx { + nameBytes := []byte(a.name) + asset := model.Asset{ + Index: a.assetID, + Params: model.AssetParams{ + Creator: accountString, + Decimals: 0, + Clawback: &accountString, + Freeze: &accountString, + Manager: &accountString, + Reserve: &accountString, + Name: &a.name, + NameB64: &nameBytes, + Total: assetTotal, + }, + } + asset.Params.DefaultFrozen = new(bool) + *(asset.Params.DefaultFrozen) = false + createdAssets = append(createdAssets, asset) + } + } + + data := model.Account{ + Address: accountString, + Amount: g.balances[idx], + AmountWithoutPendingRewards: g.balances[idx], + AppsLocalState: nil, + AppsTotalExtraPages: nil, + AppsTotalSchema: nil, + Assets: &assets, + AuthAddr: nil, + CreatedApps: nil, + CreatedAssets: &createdAssets, + Participation: nil, + PendingRewards: 0, + RewardBase: nil, + Rewards: 0, + Round: g.round - 1, + SigType: nil, + Status: "Offline", + } + + return json.NewEncoder(output).Encode(data) +} + +// Accounts is used in the runner to generate a list of addresses. +func (g *generator) Accounts() <-chan basics.Address { + results := make(chan basics.Address, 10) + go func() { + defer close(results) + for i := uint64(0); i < g.numAccounts; i++ { + results <- indexToAccount(i) + } + }() + return results +} diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go new file mode 100644 index 0000000000..49cd49d17e --- /dev/null +++ b/tools/block-generator/generator/generate_test.go @@ -0,0 +1,186 @@ +package generator + +import ( + "bytes" + "testing" + + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + "github.com/stretchr/testify/require" +) + +func makePrivateGenerator(t *testing.T) *generator { + publicGenerator, err := MakeGenerator(GenerationConfig{ + NumGenesisAccounts: 10, + GenesisAccountInitialBalance: 10000000000000000000, + PaymentTransactionFraction: 1.0, + PaymentNewAccountFraction: 1.0, + AssetCreateFraction: 1.0, + }) + require.NoError(t, err) + return publicGenerator.(*generator) +} + +func TestPaymentAcctCreate(t *testing.T) { + g := makePrivateGenerator(t) + g.generatePaymentTxnInternal(paymentAcctCreateTx, 0, 0) + require.Len(t, g.balances, int(g.config.NumGenesisAccounts+1)) +} + +func TestPaymentTransfer(t *testing.T) { + g := makePrivateGenerator(t) + g.generatePaymentTxnInternal(paymentTx, 0, 0) + require.Len(t, g.balances, int(g.config.NumGenesisAccounts)) +} + +func TestAssetXferNoAssetsOverride(t *testing.T) { + g := makePrivateGenerator(t) + + // First asset transaction must create. + actual, txn := g.generateAssetTxnInternal(assetXfer, 1, 0) + require.Equal(t, assetCreate, actual) + require.Equal(t, protocol.AssetConfigTx, txn.Type) + require.Len(t, g.assets, 0) + require.Len(t, g.pendingAssets, 1) + require.Len(t, g.pendingAssets[0].holdings, 1) + require.Len(t, g.pendingAssets[0].holders, 1) +} + +func TestAssetXferOneHolderOverride(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + + // Transfer converted to optin if there is only 1 holder. + actual, txn := g.generateAssetTxnInternal(assetXfer, 2, 0) + require.Equal(t, assetOptin, actual) + require.Equal(t, protocol.AssetTransferTx, txn.Type) + require.Len(t, g.assets, 1) + // A new holding is created, indicating the optin + require.Len(t, g.assets[0].holdings, 2) + require.Len(t, g.assets[0].holders, 2) +} + +func TestAssetCloseCreatorOverride(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + + // Instead of closing the creator, optin a new account + actual, txn := g.generateAssetTxnInternal(assetClose, 2, 0) + require.Equal(t, assetOptin, actual) + require.Equal(t, protocol.AssetTransferTx, txn.Type) + require.Len(t, g.assets, 1) + // A new holding is created, indicating the optin + require.Len(t, g.assets[0].holdings, 2) + require.Len(t, g.assets[0].holders, 2) +} + +func TestAssetOptinEveryAccountOverride(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + + // Opt all the accounts in, this also verifies that no account is opted in twice + var txn transactions.Transaction + var actual TxTypeID + for i := 2; uint64(i) <= g.numAccounts; i++ { + actual, txn = g.generateAssetTxnInternal(assetOptin, 2, uint64(1+i)) + require.Equal(t, assetOptin, actual) + require.Equal(t, protocol.AssetTransferTx, txn.Type) + require.Len(t, g.assets, 1) + require.Len(t, g.assets[0].holdings, i) + require.Len(t, g.assets[0].holders, i) + } + g.finishRound(2) + + // All accounts have opted in + require.Equal(t, g.numAccounts, uint64(len(g.assets[0].holdings))) + + // The next optin closes instead + actual, txn = g.generateAssetTxnInternal(assetOptin, 3, 0) + g.finishRound(3) + require.Equal(t, assetClose, actual) + require.Equal(t, protocol.AssetTransferTx, txn.Type) + require.Len(t, g.assets, 1) + require.Len(t, g.assets[0].holdings, int(g.numAccounts-1)) + require.Len(t, g.assets[0].holders, int(g.numAccounts-1)) +} + +func TestAssetDestroyWithHoldingsOverride(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + g.generateAssetTxnInternal(assetOptin, 2, 0) + g.finishRound(2) + g.generateAssetTxnInternal(assetXfer, 3, 0) + g.finishRound(3) + require.Len(t, g.assets[0].holdings, 2) + require.Len(t, g.assets[0].holders, 2) + + actual, txn := g.generateAssetTxnInternal(assetDestroy, 4, 0) + require.Equal(t, assetClose, actual) + require.Equal(t, protocol.AssetTransferTx, txn.Type) + require.Len(t, g.assets, 1) + require.Len(t, g.assets[0].holdings, 1) + require.Len(t, g.assets[0].holders, 1) +} + +func TestAssetTransfer(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + g.generateAssetTxnInternal(assetOptin, 2, 0) + g.finishRound(2) + g.generateAssetTxnInternal(assetXfer, 3, 0) + g.finishRound(3) + require.Greater(t, g.assets[0].holdings[1].balance, uint64(0)) +} + +func TestAssetDestroy(t *testing.T) { + g := makePrivateGenerator(t) + g.finishRound(0) + g.generateAssetTxnInternal(assetCreate, 1, 0) + g.finishRound(1) + require.Len(t, g.assets, 1) + + actual, txn := g.generateAssetTxnInternal(assetDestroy, 2, 0) + require.Equal(t, assetDestroy, actual) + require.Equal(t, protocol.AssetConfigTx, txn.Type) + require.Len(t, g.assets, 0) +} + +func TestWriteRoundZero(t *testing.T) { + g := makePrivateGenerator(t) + var data []byte + writer := bytes.NewBuffer(data) + g.WriteBlock(writer, 0) + var block rpcs.EncodedBlockCert + protocol.Decode(data, &block) + require.Len(t, block.Block.Payset, 0) +} + +func TestWriteRound(t *testing.T) { + g := makePrivateGenerator(t) + var data []byte + writer := bytes.NewBuffer(data) + g.WriteBlock(writer, 1) + var block rpcs.EncodedBlockCert + protocol.Decode(data, &block) + require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) +} + +func TestIndexToAccountAndAccountToIndex(t *testing.T) { + for i := uint64(0); i < uint64(100000); i++ { + acct := indexToAccount(i) + result := accountToIndex(acct) + require.Equal(t, i, result) + } +} diff --git a/tools/block-generator/generator/make_transactions.go b/tools/block-generator/generator/make_transactions.go new file mode 100644 index 0000000000..71e84c9d4c --- /dev/null +++ b/tools/block-generator/generator/make_transactions.go @@ -0,0 +1,81 @@ +package generator + +import ( + "encoding/binary" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/protocol" +) + +func (g *generator) makeTxnHeader(sender basics.Address, round, intra uint64) transactions.Header { + note := make([]byte, 8) + binary.LittleEndian.PutUint64(note, uint64(g.txnCounter+intra)) + + return transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: g.params.MinTxnFee}, + FirstValid: basics.Round(round), + LastValid: basics.Round(round + 1000), + GenesisID: g.genesisID, + GenesisHash: g.genesisHash, + Note: note, + } +} + +func (g *generator) makePaymentTxn(header transactions.Header, receiver basics.Address, amount uint64, closeRemainderTo basics.Address) transactions.Transaction { + return transactions.Transaction{ + Type: protocol.PaymentTx, + Header: header, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: amount}, + CloseRemainderTo: closeRemainderTo, + }, + } +} + +func (g *generator) makeAssetCreateTxn(header transactions.Header, total uint64, defaultFrozen bool, assetName string) transactions.Transaction { + return transactions.Transaction{ + Type: protocol.AssetConfigTx, + Header: header, + AssetConfigTxnFields: transactions.AssetConfigTxnFields{ + AssetParams: basics.AssetParams{ + Total: total, + DefaultFrozen: defaultFrozen, + AssetName: assetName, + Manager: header.Sender, + Freeze: header.Sender, + Clawback: header.Sender, + Reserve: header.Sender, + }, + }, + } +} + +func (g *generator) makeAssetDestroyTxn(header transactions.Header, index uint64) transactions.Transaction { + return transactions.Transaction{ + Type: protocol.AssetConfigTx, + Header: header, + AssetConfigTxnFields: transactions.AssetConfigTxnFields{ + ConfigAsset: basics.AssetIndex(index), + }, + } +} + +func (g *generator) makeAssetTransferTxn(header transactions.Header, receiver basics.Address, amount uint64, closeAssetsTo basics.Address, index uint64) transactions.Transaction { + return transactions.Transaction{ + Type: protocol.AssetTransferTx, + Header: header, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + XferAsset: basics.AssetIndex(index), + AssetAmount: amount, + AssetReceiver: receiver, + AssetCloseTo: closeAssetsTo, + }, + } +} + +func (g *generator) makeAssetAcceptanceTxn(header transactions.Header, index uint64) transactions.Transaction { + return g.makeAssetTransferTxn(header, header.Sender, 0, basics.Address{}, index) +} diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go new file mode 100644 index 0000000000..4b8d674e0e --- /dev/null +++ b/tools/block-generator/generator/server.go @@ -0,0 +1,153 @@ +package generator + +import ( + "fmt" + "net/http" + "os" + "strings" + + "github.com/algorand/go-algorand/tools/block-generator/util" + "gopkg.in/yaml.v3" +) + +func initializeConfigFile(configFile string) (config GenerationConfig, err error) { + data, err := os.ReadFile(configFile) + if err != nil { + return + } + yaml.Unmarshal(data, &config) + return +} + +// MakeServer configures http handlers. Returns the http server. +func MakeServer(configFile string, addr string) (*http.Server, Generator) { + noOp := func(next http.Handler) http.Handler { + return next + } + return MakeServerWithMiddleware(configFile, addr, noOp) +} + +// BlocksMiddleware is a middleware for the blocks endpoint. +type BlocksMiddleware func(next http.Handler) http.Handler + +// MakeServerWithMiddleware allows injecting a middleware for the blocks handler. +// This is needed to simplify tests by stopping block production while validation +// is done on the data. +func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware BlocksMiddleware) (*http.Server, Generator) { + config, err := initializeConfigFile(configFile) + util.MaybeFail(err, "problem loading config file. Use '--config' or create a config file.") + + gen, err := MakeGenerator(config) + util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile) + + mux := http.NewServeMux() + mux.HandleFunc("/", help) + mux.Handle("/v2/blocks/", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen)))) + mux.HandleFunc("/v2/accounts/", getAccountHandler(gen)) + mux.HandleFunc("/genesis", getGenesisHandler(gen)) + mux.HandleFunc("/report", getReportHandler(gen)) + mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) + + return &http.Server{ + Addr: addr, + Handler: mux, + }, gen +} + +func help(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Use /v2/blocks/:blocknum: to get a block.") +} + +func maybeWriteError(w http.ResponseWriter, err error) { + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +func getReportHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + maybeWriteError(w, gen.WriteReport(w)) + } +} + +func getStatusWaitHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + maybeWriteError(w, gen.WriteStatus(w)) + } +} + +func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + maybeWriteError(w, gen.WriteGenesis(w)) + } +} + +func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + // The generator doesn't actually care about the block... + round, err := parseRound(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + maybeWriteError(w, gen.WriteBlock(w, round)) + } +} + +func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + // The generator doesn't actually care about the block... + account, err := parseAccount(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + maybeWriteError(w, gen.WriteAccount(w, account)) + } +} + +const blockQueryPrefix = "/v2/blocks/" +const blockQueryBlockIdx = len(blockQueryPrefix) +const accountsQueryPrefix = "/v2/accounts/" +const accountsQueryAccountIdx = len(accountsQueryPrefix) + +func parseRound(path string) (uint64, error) { + if !strings.HasPrefix(path, blockQueryPrefix) { + return 0, fmt.Errorf("not a blocks query: %s", path) + } + + result := uint64(0) + pathlen := len(path) + + if pathlen == blockQueryBlockIdx { + return 0, fmt.Errorf("no block in path") + } + + for i := blockQueryBlockIdx; i < pathlen; i++ { + if path[i] < '0' || path[i] > '9' { + if i == blockQueryBlockIdx { + return 0, fmt.Errorf("no block in path") + } + break + } + result = (uint64(10) * result) + uint64(int(path[i])-'0') + } + return result, nil +} + +func parseAccount(path string) (string, error) { + if !strings.HasPrefix(path, accountsQueryPrefix) { + return "", fmt.Errorf("not a accounts query: %s", path) + } + + pathlen := len(path) + + if pathlen == accountsQueryAccountIdx { + return "", fmt.Errorf("no address in path") + } + + return path[accountsQueryAccountIdx:], nil +} diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go new file mode 100644 index 0000000000..039dd3885f --- /dev/null +++ b/tools/block-generator/generator/server_test.go @@ -0,0 +1,88 @@ +package generator + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitConfigFile(t *testing.T) { + config, err := initializeConfigFile("../test_config.yml") + require.NoError(t, err) + require.Equal(t, uint64(10), config.NumGenesisAccounts) + require.Equal(t, float32(0.25), config.AssetCloseFraction) + require.Equal(t, float32(0.0), config.AssetDestroyFraction) +} + +func TestInitConfigFileNotExist(t *testing.T) { + _, err := initializeConfigFile("this_is_not_a_config_file") + + if _, ok := err.(*os.PathError); !ok { + require.Fail(t, "This should generate a path error") + } +} + +func TestParseRound(t *testing.T) { + var testcases = []struct { + name string + url string + expectedRound uint64 + err string + }{ + { + name: "no block", + url: "/v2/blocks/", + expectedRound: 0, + err: "no block in path", + }, + { + name: "no block 2", + url: "/v2/blocks/?nothing", + expectedRound: 0, + err: "no block in path", + }, + { + name: "invalid prefix", + url: "/v2/wrong/prefix/1", + expectedRound: 0, + err: "not a blocks query", + }, + { + name: "normal one digit", + url: fmt.Sprintf("%s1", blockQueryPrefix), + expectedRound: 1, + err: "", + }, + { + name: "normal long number", + url: fmt.Sprintf("%s12345678", blockQueryPrefix), + expectedRound: 12345678, + err: "", + }, + { + name: "with query parameters", + url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), + expectedRound: 1234, + err: "", + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + round, err := parseRound(testcase.url) + if len(testcase.err) == 0 { + msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%d' received error: %v", + testcase.url, testcase.expectedRound, err) + require.NoError(t, err, msg) + assert.Equal(t, testcase.expectedRound, round) + } else { + require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err)) + require.True(t, strings.Contains(err.Error(), testcase.err)) + } + }) + } +} diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go new file mode 100644 index 0000000000..fed2c1ec8c --- /dev/null +++ b/tools/block-generator/generator/utils.go @@ -0,0 +1,29 @@ +package generator + +import ( + "fmt" + "math/rand" +) + +func weightedSelection(weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) { + return weightedSelectionInternal(rand.Float32(), weights, options, defaultOption) +} + +func weightedSelectionInternal(selectionNumber float32, weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) { + if len(weights) != len(options) { + err = fmt.Errorf("number of weights must equal number of options: %d != %d", len(weights), len(options)) + return + } + + total := float32(0) + for i := 0; i < len(weights); i++ { + if selectionNumber-total < weights[i] { + selection = options[i] + return + } + total += weights[i] + } + + selection = defaultOption + return +} diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go new file mode 100644 index 0000000000..09dbb8b05c --- /dev/null +++ b/tools/block-generator/generator/utils_test.go @@ -0,0 +1,90 @@ +package generator + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWeightedSelectionInternalBadInput(t *testing.T) { + weights := []float32{0.10, 0.30} + options := []interface{}{"10"} + _, err := weightedSelectionInternal(0, weights, options, nil) + require.EqualError(t, err, "number of weights must equal number of options: 2 != 1") +} + +func TestWeightedSelectionInternal(t *testing.T) { + weights := []float32{0.10, 0.30, 0.60} + options := []interface{}{"10", "30", "60"} + + testcases := []struct { + selectionNum float32 + expected interface{} + }{ + { + selectionNum: 0.0, + expected: options[0], + }, + { + selectionNum: 0.099, + expected: options[0], + }, + { + selectionNum: 0.1, + expected: options[1], + }, + { + selectionNum: 0.399, + expected: options[1], + }, + { + selectionNum: 0.4, + expected: options[2], + }, + { + selectionNum: 0.999, + expected: options[2], + }, + } + + for _, test := range testcases { + name := fmt.Sprintf("selectionNum %f - expected %v", test.selectionNum, test.expected) + t.Run(name, func(t *testing.T) { + actual, err := weightedSelectionInternal(test.selectionNum, weights, options, nil) + require.NoError(t, err) + require.Equal(t, test.expected, actual) + }) + } +} + +func TestWeightedSelection(t *testing.T) { + weights := []float32{0.10, 0.30, 0.60} + options := []interface{}{"10", "30", "60"} + selections := make(map[interface{}]int) + + for i := 0; i < 100; i++ { + selected, err := weightedSelection(weights, options, nil) + require.NoError(t, err) + selections[selected]++ + } + + assert.Less(t, selections[options[0]], selections[options[1]]) + assert.Less(t, selections[options[1]], selections[options[2]]) +} + +func TestWeightedSelectionOutOfRange(t *testing.T) { + weights := []float32{0.1} + options := []interface{}{"1"} + defaultOption := "DEFAULT!" + + for i := 0; i < 10000; i++ { + selection, err := weightedSelection(weights, options, defaultOption) + require.NoError(t, err) + if selection == defaultOption { + return + } + } + assert.Fail(t, "Expected an out of range error by this point.") +} diff --git a/tools/block-generator/main.go b/tools/block-generator/main.go new file mode 100644 index 0000000000..4a9e1b7c33 --- /dev/null +++ b/tools/block-generator/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/algorand/go-algorand/tools/block-generator/core" + +func main() { + core.BlockGenerator.Execute() +} diff --git a/tools/block-generator/metrics/metrics.go b/tools/block-generator/metrics/metrics.go new file mode 100644 index 0000000000..5645023149 --- /dev/null +++ b/tools/block-generator/metrics/metrics.go @@ -0,0 +1,26 @@ +package metrics + +// Prometheus metrics collected in Conduit. +const ( + BlockImportTimeName = "import_time_sec" + ImportedTxnsPerBlockName = "imported_tx_per_block" + ImportedRoundGaugeName = "imported_round" + GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" + ImportedTxnsName = "imported_txns" + ImporterTimeName = "importer_time_sec" + ProcessorTimeName = "processor_time_sec" + ExporterTimeName = "exporter_time_sec" + PipelineRetryCountName = "pipeline_retry_count" +) + +// AllMetricNames is a reference for all the custom metric names. +var AllMetricNames = []string{ + BlockImportTimeName, + ImportedTxnsPerBlockName, + ImportedRoundGaugeName, + GetAlgodRawBlockTimeName, + ImporterTimeName, + ProcessorTimeName, + ExporterTimeName, + PipelineRetryCountName, +} diff --git a/tools/block-generator/run_generator.sh b/tools/block-generator/run_generator.sh new file mode 100755 index 0000000000..47e87b2e0d --- /dev/null +++ b/tools/block-generator/run_generator.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Demonstrate how to run the generator and connect it to indexer. + +set -e + +POSTGRES_CONTAINER=generator-test-container +POSTGRES_PORT=15432 +POSTGRES_DATABASE=generator_db +CONFIG=${1:-"$(dirname $0)/test_config.yml"} +echo "Using config file: $CONFIG" + +function start_postgres() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true + + # Start postgres container... + docker run \ + -d \ + --name $POSTGRES_CONTAINER \ + -e POSTGRES_USER=algorand \ + -e POSTGRES_PASSWORD=algorand \ + -e PGPASSWORD=algorand \ + -p $POSTGRES_PORT:5432 \ + postgres + + sleep 5 + + docker exec -it $POSTGRES_CONTAINER psql -Ualgorand -c "create database $POSTGRES_DATABASE" +} + +function shutdown() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true + kill -9 $GENERATOR_PID +} + +trap shutdown EXIT + +echo "Building generator." +pushd $(dirname "$0") > /dev/null +go build +cd ../.. > /dev/null +echo "Building indexer." +make +popd +echo "Starting postgres container." +start_postgres +echo "Starting block generator (see generator.log)" +$(dirname "$0")/block-generator daemon --port 11111 --config "${CONFIG}" & +GENERATOR_PID=$! +echo "Starting indexer" +$(dirname "$0")/../../cmd/algorand-indexer/algorand-indexer daemon \ + -S localhost:8980 \ + --algod-net localhost:11111 \ + --algod-token security-is-our-number-one-priority \ + --metrics-mode VERBOSE \ + -P "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" diff --git a/tools/block-generator/run_postgres.sh b/tools/block-generator/run_postgres.sh new file mode 100755 index 0000000000..c6a967132b --- /dev/null +++ b/tools/block-generator/run_postgres.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# This script is useful if you want to launch the runner +# in a debugger. Simply start this script and run with: +# ./block-generator runner \ +# -d 5s \ +# -i ./../algorand-indexer/algorand-indexer \ +# -c "host=localhost user=algorand password=algorand dbname=algorand port=15432 sslmode=disable" \ +# -r results \ +# -s scenarios/config.payment.small.yml + +set -e + +POSTGRES_CONTAINER=generator-test-container +POSTGRES_PORT=15432 +POSTGRES_DATABASE=generator_db +CONFIG=${1:-"$(dirname $0)/test_config.yml"} +echo "Using config file: $CONFIG" + +function start_postgres() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true + + # Start postgres container... + docker run \ + -d \ + --name $POSTGRES_CONTAINER \ + -e POSTGRES_USER=algorand \ + -e POSTGRES_PASSWORD=algorand \ + -e PGPASSWORD=algorand \ + -p $POSTGRES_PORT:5432 \ + postgres + + sleep 5 + + docker exec -it $POSTGRES_CONTAINER psql -Ualgorand -c "create database $POSTGRES_DATABASE" +} + +function shutdown() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true +} + +trap shutdown EXIT + +pushd $(dirname "$0") > /dev/null +echo "Starting postgres container at: \n\t\"host=localhost user=algorand password=algorand dbname=algorand port=15432\"" +start_postgres +echo "Sleeping, use Ctrl-C to end test." +sleep 1000000000000000 + diff --git a/tools/block-generator/run_runner.sh b/tools/block-generator/run_runner.sh new file mode 100755 index 0000000000..d90749b96b --- /dev/null +++ b/tools/block-generator/run_runner.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Demonstrate how to run the block-generator runner. + +set -e + +POSTGRES_CONTAINER=generator-test-container +POSTGRES_PORT=15432 +POSTGRES_DATABASE=generator_db +CONFIG=${1:-"$(dirname $0)/test_config.yml"} +echo "Using config file: $CONFIG" + +function start_postgres() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true + + # Start postgres container... + docker run \ + -d \ + --name $POSTGRES_CONTAINER \ + -e POSTGRES_USER=algorand \ + -e POSTGRES_PASSWORD=algorand \ + -e PGPASSWORD=algorand \ + -p $POSTGRES_PORT:5432 \ + postgres + + sleep 5 + + docker exec -it $POSTGRES_CONTAINER psql -Ualgorand -c "create database $POSTGRES_DATABASE" +} + +function shutdown() { + docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true +} + +trap shutdown EXIT + +rm -rf OUTPUT_RUN_RUNNER_TEST > /dev/null 2>&1 +echo "Building generator." +pushd $(dirname "$0") > /dev/null +go build +cd ../.. > /dev/null +echo "Building indexer." +make +popd +echo "Starting postgres container." +start_postgres +echo "Starting test runner" +$(dirname "$0")/block-generator runner \ + --indexer-binary ../algorand-indexer/algorand-indexer \ + --report-directory OUTPUT_RUN_RUNNER_TEST \ + --test-duration 30s \ + --log-level trace \ + --postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \ + --scenario ${CONFIG} diff --git a/tools/block-generator/run_tests.sh b/tools/block-generator/run_tests.sh new file mode 100755 index 0000000000..0a1c6df49e --- /dev/null +++ b/tools/block-generator/run_tests.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +CONNECTION_STRING="" +INDEXER_BINARY="" +REPORT_DIR="" +DURATION="1h" +LOG_LEVEL="error" +SCENARIOS="" + +help() { + echo "Usage:" + echo " -v|--verbose enable verbose script output." + echo " -c|--connection-string" + echo " PostgreSQL connection string." + echo " -i|--indexer path to indexer binary." + echo " -s|--scenarios path to indexer test scenarios." + echo " -r|--report-dir directory where the report should be written." + echo " -d|--duration test duration." + echo " -l|--level log level to pass to Indexer." + echo " -g|--generator use a different indexer binary to run the generator." + exit +} + +while :; do + case "${1-}" in + -h | --help) help ;; + -v | --verbose) set -x ;; + -c | --connection-string) + CONNECTION_STRING="${2-}" + shift + ;; + -g | --generator) + GENERATOR_BINARY="${2-}" + shift + ;; + -i | --indexer) + INDEXER_BINARY="${2-}" + shift + ;; + -r | --report-dir) + REPORT_DIR="${2-}" + shift + ;; + -s | --scenarios) + SCENARIOS="${2-}" + shift + ;; + -d | --duration) + DURATION="${2-}" + shift + ;; + -l | --level) + LOG_LEVEL="${2-}" + shift + ;; + -?*) echo "Unknown option: $1" && exit 1;; + *) break ;; + esac + shift +done + +args=("$@") + +if [ -z "$CONNECTION_STRING" ]; then + echo "Missing required connection string parameter (-c / --connection-string)." + exit 1 +fi + +if [ -z "$INDEXER_BINARY" ]; then + echo "Missing required indexer binary parameter (-i / --indexer)." + exit 1 +fi + +if [ -z "$SCENARIOS" ]; then + echo "Missing required indexer test scenario parameter (-s / --scenarios)." + exit 1 +fi + +if [ -z "$GENERATOR_BINARY" ]; then + echo "Using indexer binary for generator, override with (-g / --generator)." + GENERATOR_BINARY="$INDEXER_BINARY" +fi + +echo "Running with binary: $INDEXER_BINARY" +echo "Report directory: $REPORT_DIR" +echo "Duration: $DURATION" +echo "Log Level: $LOG_LEVEL" + +"$GENERATOR_BINARY" \ + util block-generator runner \ + -i "$INDEXER_BINARY" \ + -s "$SCENARIOS" \ + -d "$DURATION" \ + -c "$CONNECTION_STRING" \ + --report-directory "$REPORT_DIR" \ + --log-level "$LOG_LEVEL" \ + --reset + diff --git a/tools/block-generator/runner/metrics_collector.go b/tools/block-generator/runner/metrics_collector.go new file mode 100644 index 0000000000..7c89aed02a --- /dev/null +++ b/tools/block-generator/runner/metrics_collector.go @@ -0,0 +1,72 @@ +package runner + +import ( + "bufio" + "fmt" + "net/http" + "strings" + "time" +) + +// MetricsCollector queries a /metrics endpoint for prometheus style metrics and saves metrics matching a pattern. +type MetricsCollector struct { + // MetricsURL where metrics can be queried. + MetricsURL string + // Data is all of the results. + Data []Entry +} + +// Entry is the raw data pulled from the endpoint along with a timestamp. +type Entry struct { + Timestamp time.Time + Data []string +} + +// Collect fetches the metrics. +func (r *MetricsCollector) Collect(substrings ...string) error { + metrics, err := r.getMetrics(substrings...) + if err != nil { + return err + } + + if len(metrics) > 0 { + entry := Entry{ + Timestamp: time.Now(), + Data: metrics, + } + r.Data = append(r.Data, entry) + } + + return nil +} + +func (r MetricsCollector) getMetrics(substrings ...string) (result []string, err error) { + resp, err := http.Get(r.MetricsURL) + if err != nil { + err = fmt.Errorf("unable to read metrics url '%s'", r.MetricsURL) + return + } + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + str := scanner.Text() + + if strings.HasPrefix(str, "#") { + continue + } + + for _, substring := range substrings { + if strings.Contains(str, substring) { + result = append(result, str) + break + } + } + } + + if scanner.Err() != nil { + err = fmt.Errorf("problem reading metrics response: %w", scanner.Err()) + } + + return +} diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go new file mode 100644 index 0000000000..47a2a6a985 --- /dev/null +++ b/tools/block-generator/runner/run.go @@ -0,0 +1,331 @@ +package runner + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/algorand/go-algorand/tools/block-generator/generator" + "github.com/algorand/go-algorand/tools/block-generator/metrics" + "github.com/algorand/go-algorand/tools/block-generator/util" +) + +// Args are all the things needed to run a performance test. +type Args struct { + // Path is a directory when passed to RunBatch, otherwise a file path. + Path string + IndexerBinary string + IndexerPort uint64 + PostgresConnectionString string + CPUProfilePath string + RunDuration time.Duration + LogLevel string + ReportDirectory string + ResetReportDir bool + RunValidation bool + KeepDataDir bool +} + +// Run is a publi8c helper to run the tests. +// The test will run against the generator configuration file specified by 'args.Path'. +// If 'args.Path' is a directory it should contain generator configuration files, a test will run using each file. +func Run(args Args) error { + if _, err := os.Stat(args.ReportDirectory); !os.IsNotExist(err) { + if args.ResetReportDir { + fmt.Printf("Resetting existing report directory '%s'\n", args.ReportDirectory) + if err := os.RemoveAll(args.ReportDirectory); err != nil { + return fmt.Errorf("failed to reset report directory: %w", err) + } + } else { + return fmt.Errorf("report directory '%s' already exists", args.ReportDirectory) + } + } + os.Mkdir(args.ReportDirectory, os.ModeDir|os.ModePerm) + + defer fmt.Println("Done running tests!") + return filepath.Walk(args.Path, func(path string, info os.FileInfo, err error) error { + // Ignore the directory + if info.IsDir() { + return nil + } + runnerArgs := args + runnerArgs.Path = path + fmt.Printf("Running test for configuration '%s'\n", path) + return runnerArgs.run() + }) +} + +func (r *Args) run() error { + baseName := filepath.Base(r.Path) + baseNameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName)) + reportfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.report", baseNameNoExt)) + //logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.indexer-log", baseNameNoExt)) + dataDir := path.Join(r.ReportDirectory, fmt.Sprintf("%s_data", baseNameNoExt)) + if !r.KeepDataDir { + defer os.RemoveAll(dataDir) + } + + // This middleware allows us to lock the block endpoint + var freezeMutex sync.Mutex + blockMiddleware := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + freezeMutex.Lock() + defer freezeMutex.Unlock() + next.ServeHTTP(w, r) + }) + } + // Start services + algodNet := fmt.Sprintf("localhost:%d", 11112) + indexerNet := fmt.Sprintf("localhost:%d", r.IndexerPort) + generatorShutdownFunc, _ := startGenerator(r.Path, algodNet, blockMiddleware) + defer func() { + // Shutdown generator. + if err := generatorShutdownFunc(); err != nil { + fmt.Printf("Failed to shutdown generator: %s\n", err) + } + }() + + //indexerShutdownFunc, err := startIndexer(dataDir, logfile, r.LogLevel, r.IndexerBinary, algodNet, indexerNet, r.PostgresConnectionString, r.CPUProfilePath) + //if err != nil { + // return fmt.Errorf("failed to start indexer: %w", err) + //} + //defer func() { + // // Shutdown indexer + // if err := indexerShutdownFunc(); err != nil { + // fmt.Printf("Failed to shutdown indexer: %s\n", err) + // } + //}() + + // Create the report file + report, err := os.Create(reportfile) + if err != nil { + return fmt.Errorf("unable to create report: %w", err) + } + defer report.Close() + + // Run the test, collecting results. + if err := r.runTest(report, indexerNet, algodNet); err != nil { + return err + } + + return nil +} + +type metricType int + +const ( + rate metricType = iota + intTotal + floatTotal +) + +// Helper to record metrics. Supports rates (sum/count) and counters. +func recordDataToFile(start time.Time, entry Entry, prefix string, out *os.File) error { + var writeErrors []string + var writeErr error + record := func(prefix2, name string, t metricType) { + key := fmt.Sprintf("%s%s_%s", prefix, prefix2, name) + if err := recordMetricToFile(entry, key, name, t, out); err != nil { + writeErr = err + writeErrors = append(writeErrors, name) + } + } + + record("_average", metrics.BlockImportTimeName, rate) + record("_cumulative", metrics.BlockImportTimeName, floatTotal) + record("_average", metrics.ImportedTxnsPerBlockName, rate) + record("_cumulative", metrics.ImportedTxnsPerBlockName, intTotal) + record("", metrics.ImportedRoundGaugeName, intTotal) + + if len(writeErrors) > 0 { + return fmt.Errorf("error writing metrics (%s): %w", strings.Join(writeErrors, ", "), writeErr) + } + + // Calculate import transactions per second. + totalTxn, err := getMetric(entry, metrics.ImportedTxnsPerBlockName, false) + if err != nil { + return err + } + + importTimeS, err := getMetric(entry, metrics.BlockImportTimeName, false) + if err != nil { + return err + } + tps := totalTxn / importTimeS + key := "overall_transactions_per_second" + msg := fmt.Sprintf("%s_%s:%.2f\n", prefix, key, tps) + if _, err := out.WriteString(msg); err != nil { + return fmt.Errorf("unable to write metric '%s': %w", key, err) + } + + // Uptime + key = "uptime_seconds" + msg = fmt.Sprintf("%s_%s:%.2f\n", prefix, key, time.Since(start).Seconds()) + if _, err := out.WriteString(msg); err != nil { + return fmt.Errorf("unable to write metric '%s': %w", key, err) + } + + return nil +} + +func recordMetricToFile(entry Entry, outputKey, metricSuffix string, t metricType, out *os.File) error { + value, err := getMetric(entry, metricSuffix, t == rate) + if err != nil { + return err + } + + var msg string + if t == intTotal { + msg = fmt.Sprintf("%s:%d\n", outputKey, uint64(value)) + } else { + msg = fmt.Sprintf("%s:%.2f\n", outputKey, value) + } + + if _, err := out.WriteString(msg); err != nil { + return fmt.Errorf("unable to write metric '%s': %w", outputKey, err) + } + + return nil +} + +func getMetric(entry Entry, suffix string, rateMetric bool) (float64, error) { + total := 0.0 + sum := 0.0 + count := 0.0 + hasSum := false + hasCount := false + hasTotal := false + + for _, metric := range entry.Data { + var err error + + if strings.Contains(metric, suffix) { + split := strings.Split(metric, " ") + if len(split) != 2 { + return 0.0, fmt.Errorf("unknown metric format, expected 'key value' received: %s", metric) + } + + // Check for _sum / _count for summary (rateMetric) metrics. + // Otherwise grab the total value. + if strings.HasSuffix(split[0], "_sum") { + sum, err = strconv.ParseFloat(split[1], 64) + hasSum = true + } else if strings.HasSuffix(split[0], "_count") { + count, err = strconv.ParseFloat(split[1], 64) + hasCount = true + } else if strings.HasSuffix(split[0], suffix) { + total, err = strconv.ParseFloat(split[1], 64) + hasTotal = true + } + + if err != nil { + return 0.0, fmt.Errorf("unable to parse metric '%s': %w", metric, err) + } + + if rateMetric && hasSum && hasCount { + return sum / count, nil + } else if !rateMetric { + if hasSum { + return sum, nil + } + if hasTotal { + return total, nil + } + } + } + } + + return 0.0, fmt.Errorf("metric incomplete or not found: %s", suffix) +} + +// Run the test for 'RunDuration', collect metrics and write them to the 'ReportDirectory' +func (r *Args) runTest(report *os.File, indexerURL string, generatorURL string) error { + collector := &MetricsCollector{MetricsURL: fmt.Sprintf("http://%s/metrics", indexerURL)} + + // Run for r.RunDuration + start := time.Now() + count := 1 + for time.Since(start) < r.RunDuration { + time.Sleep(r.RunDuration / 10) + + if err := collector.Collect(metrics.AllMetricNames...); err != nil { + return fmt.Errorf("problem collecting metrics (%d / %s): %w", count, time.Since(start), err) + } + count++ + } + if err := collector.Collect(metrics.AllMetricNames...); err != nil { + return fmt.Errorf("problem collecting final metrics (%d / %s): %w", count, time.Since(start), err) + } + + // Collect results. + durationStr := fmt.Sprintf("test_duration_seconds:%d\ntest_duration_actual_seconds:%f\n", + uint64(r.RunDuration.Seconds()), + time.Since(start).Seconds()) + if _, err := report.WriteString(durationStr); err != nil { + return fmt.Errorf("unable to write duration metric: %w", err) + } + + resp, err := http.Get(fmt.Sprintf("http://%s/report", generatorURL)) + if err != nil { + return fmt.Errorf("generator report query failed") + } + defer resp.Body.Close() + var generatorReport generator.Report + if err = json.NewDecoder(resp.Body).Decode(&generatorReport); err != nil { + return fmt.Errorf("problem decoding generator report: %w", err) + } + for metric, entry := range generatorReport { + // Skip this one + if metric == "genesis" { + continue + } + str := fmt.Sprintf("transaction_%s_total:%d\n", metric, entry.GenerationCount) + if _, err = report.WriteString(str); err != nil { + return fmt.Errorf("unable to write transaction_count metric: %w", err) + } + } + + // Record a rate from one of the first data points. + if len(collector.Data) > 5 { + if err = recordDataToFile(start, collector.Data[2], "early", report); err != nil { + return err + } + } + + // Also record the final metrics. + if err = recordDataToFile(start, collector.Data[len(collector.Data)-1], "final", report); err != nil { + return err + } + + return nil +} + +// startGenerator starts the generator server. +func startGenerator(configFile string, addr string, blockMiddleware func(http.Handler) http.Handler) (func() error, generator.Generator) { + // Start generator. + server, generator := generator.MakeServerWithMiddleware(configFile, addr, blockMiddleware) + + // Start the server + go func() { + // always returns error. ErrServerClosed on graceful close + if err := server.ListenAndServe(); err != http.ErrServerClosed { + util.MaybeFail(err, "ListenAndServe() failure to start with config file '%s'", configFile) + } + }() + + return func() error { + // Shutdown blocks until the server has stopped. + if err := server.Shutdown(context.Background()); err != nil { + return fmt.Errorf("failed during generator graceful shutdown: %w", err) + } + return nil + }, generator +} diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go new file mode 100644 index 0000000000..c0bbf7cbe1 --- /dev/null +++ b/tools/block-generator/runner/runner.go @@ -0,0 +1,45 @@ +package runner + +import ( + "fmt" + "math/rand" + "time" + + "github.com/spf13/cobra" +) + +// RunnerCmd launches the block-generator test suite runner. +var RunnerCmd *cobra.Command + +func init() { + rand.Seed(12345) + var runnerArgs Args + + RunnerCmd = &cobra.Command{ + Use: "runner", + Short: "Run test suite and collect results.", + Long: "Run an automated test suite using the block-generator daemon and a provided algorand-indexer binary. Results are captured to a specified output directory.", + Run: func(cmd *cobra.Command, args []string) { + if err := Run(runnerArgs); err != nil { + fmt.Println(err) + } + }, + } + + RunnerCmd.Flags().StringVarP(&runnerArgs.Path, "scenario", "s", "", "Directory containing scenarios, or specific scenario file.") + RunnerCmd.Flags().StringVarP(&runnerArgs.IndexerBinary, "indexer-binary", "i", "", "Path to indexer binary.") + RunnerCmd.Flags().Uint64VarP(&runnerArgs.IndexerPort, "indexer-port", "p", 4010, "Port to start the server at. This is useful if you have a prometheus server for collecting additional data.") + RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") + RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") + RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") + RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Indexer. [error, warn, info, debug, trace]") + RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Indexer writes its CPU profile.") + RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.") + RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") + RunnerCmd.Flags().BoolVarP(&runnerArgs.KeepDataDir, "keep-data-dir", "k", false, "If set the validator will not delete the data directory after tests complete.") + + RunnerCmd.MarkFlagRequired("scenario") + RunnerCmd.MarkFlagRequired("indexer-binary") + RunnerCmd.MarkFlagRequired("postgres-connection-string") + RunnerCmd.MarkFlagRequired("report-directory") +} diff --git a/tools/block-generator/scenarios/config.asset.close.yml b/tools/block-generator/scenarios/config.asset.close.yml new file mode 100644 index 0000000000..9a8b86bf5d --- /dev/null +++ b/tools/block-generator/scenarios/config.asset.close.yml @@ -0,0 +1,16 @@ +name: "Asset Close" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 5000 + +# transaction distribution +tx_asset_fraction: 1.0 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.50 +asset_close_fraction: 0.40 +asset_xfer_fraction: 0.099 + +# pay config must equal 1.0 +pay_xfer_fraction: 1.0 diff --git a/tools/block-generator/scenarios/config.asset.destroy.yml b/tools/block-generator/scenarios/config.asset.destroy.yml new file mode 100644 index 0000000000..953572efb4 --- /dev/null +++ b/tools/block-generator/scenarios/config.asset.destroy.yml @@ -0,0 +1,16 @@ +name: "Asset Destroy" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 5000 + +# transaction distribution +tx_asset_fraction: 1.0 + +# asset config +asset_create_fraction: 0.8 +asset_destroy_fraction: 0.2 +asset_optin_fraction: 0.0 +asset_close_fraction: 0.0 + +# pay config must equal 1.0 +pay_xfer_fraction: 1.0 diff --git a/tools/block-generator/scenarios/config.asset.xfer.yml b/tools/block-generator/scenarios/config.asset.xfer.yml new file mode 100644 index 0000000000..3e6643ba34 --- /dev/null +++ b/tools/block-generator/scenarios/config.asset.xfer.yml @@ -0,0 +1,15 @@ +name: "Asset Xfer" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 5000 + +# transaction distribution +tx_asset_fraction: 1.0 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.10 +asset_xfer_fraction: 0.899 + +# pay config must equal 1.0 +pay_xfer_fraction: 1.0 diff --git a/tools/block-generator/scenarios/config.mixed.jumbo.yml b/tools/block-generator/scenarios/config.mixed.jumbo.yml new file mode 100644 index 0000000000..d02ee4045d --- /dev/null +++ b/tools/block-generator/scenarios/config.mixed.jumbo.yml @@ -0,0 +1,19 @@ +name: "Mixed (jumbo)" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 19999 + +# transaction distribution +tx_pay_fraction: 0.3 +tx_asset_fraction: 0.7 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.1 +asset_close_fraction: 0.05 +asset_xfer_fraction: 0.849 +asset_delete_fraction: 0 diff --git a/tools/block-generator/scenarios/config.mixed.yml b/tools/block-generator/scenarios/config.mixed.yml new file mode 100644 index 0000000000..d6e1eea606 --- /dev/null +++ b/tools/block-generator/scenarios/config.mixed.yml @@ -0,0 +1,19 @@ +name: "Mixed" +genesis_accounts: 10000 +genesis_account_balance: 1000000000000 +tx_per_block: 5000 + +# transaction distribution +tx_pay_fraction: 0.3 +tx_asset_fraction: 0.7 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.1 +asset_close_fraction: 0.05 +asset_xfer_fraction: 0.849 +asset_delete_fraction: 0 diff --git a/tools/block-generator/scenarios/config.payment.full.yml b/tools/block-generator/scenarios/config.payment.full.yml new file mode 100644 index 0000000000..22aa0e9701 --- /dev/null +++ b/tools/block-generator/scenarios/config.payment.full.yml @@ -0,0 +1,14 @@ +name: "Pay (full)" +genesis_accounts: 10 +genesis_account_balance: 1000000000000000 +tx_per_block: 5000 + +# transaction distribution +tx_pay_fraction: 1.0 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config must be 1.0 +asset_create_fraction: 1.0 diff --git a/tools/block-generator/scenarios/config.payment.jumbo.yml b/tools/block-generator/scenarios/config.payment.jumbo.yml new file mode 100644 index 0000000000..2d1f0dba2f --- /dev/null +++ b/tools/block-generator/scenarios/config.payment.jumbo.yml @@ -0,0 +1,14 @@ +name: "Pay (jumbo)" +genesis_accounts: 10 +genesis_account_balance: 1000000000000000 +tx_per_block: 19999 + +# transaction distribution +tx_pay_fraction: 1.0 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config must be 1.0 +asset_create_fraction: 1.0 diff --git a/tools/block-generator/scenarios/config.payment.small.yml b/tools/block-generator/scenarios/config.payment.small.yml new file mode 100644 index 0000000000..e250b2f016 --- /dev/null +++ b/tools/block-generator/scenarios/config.payment.small.yml @@ -0,0 +1,14 @@ +name: "Pay (small)" +genesis_accounts: 10 +genesis_account_balance: 1000000000000000 +tx_per_block: 100 + +# transaction distribution +tx_pay_fraction: 1.0 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config must be 1.0 +asset_create_fraction: 1.0 diff --git a/tools/block-generator/test_config.yml b/tools/block-generator/test_config.yml new file mode 100644 index 0000000000..6d411e9ad4 --- /dev/null +++ b/tools/block-generator/test_config.yml @@ -0,0 +1,23 @@ +name: Test Config +# genesis accounts +genesis_accounts: 10 +genesis_account_balance: 40000000000000 + +tx_per_block: 10 + +# every 100 payment transactions, create an account +tx_pay_fraction: 0.5 +tx_asset_fraction: 0.5 + +# transaction distribution +pay_acct_create_fraction: 0.5 +pay_xfer_fraction: 0.5 + +# asset distribution +asset_create_fraction: 0.25 +asset_optin_fraction: 0.25 +asset_close_fraction: 0.25 +asset_xfer_fraction: 0.25 +asset_destroy_fraction: 0.0 + + diff --git a/tools/block-generator/util/util.go b/tools/block-generator/util/util.go new file mode 100644 index 0000000000..bead732f66 --- /dev/null +++ b/tools/block-generator/util/util.go @@ -0,0 +1,16 @@ +package util + +import ( + "fmt" + "os" +) + +// MaybeFail exits if there was an error. +func MaybeFail(err error, errfmt string, params ...interface{}) { + if err == nil { + return + } + fmt.Fprintf(os.Stderr, errfmt, params...) + fmt.Fprintf(os.Stderr, "\nError: %v\n", err) + os.Exit(1) +} From 48711f199ef932036a606a8b2f17dcb2c8ed0639 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 30 Mar 2023 13:46:06 -0400 Subject: [PATCH 02/16] add license, reviewdog fix --- tools/block-generator/core/commands.go | 16 ++++++++++ tools/block-generator/generator/daemon.go | 16 ++++++++++ tools/block-generator/generator/generate.go | 16 ++++++++++ .../generator/generate_test.go | 29 +++++++++++++++++++ .../generator/make_transactions.go | 16 ++++++++++ tools/block-generator/generator/server.go | 16 ++++++++++ .../block-generator/generator/server_test.go | 19 ++++++++++++ tools/block-generator/generator/utils.go | 16 ++++++++++ tools/block-generator/generator/utils_test.go | 20 +++++++++++++ tools/block-generator/main.go | 16 ++++++++++ tools/block-generator/metrics/metrics.go | 16 ++++++++++ .../runner/metrics_collector.go | 16 ++++++++++ tools/block-generator/runner/run.go | 16 ++++++++++ tools/block-generator/runner/runner.go | 16 ++++++++++ tools/block-generator/util/util.go | 16 ++++++++++ 15 files changed, 260 insertions(+) diff --git a/tools/block-generator/core/commands.go b/tools/block-generator/core/commands.go index 8fb969c294..af64c8a5d7 100644 --- a/tools/block-generator/core/commands.go +++ b/tools/block-generator/core/commands.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package core import ( diff --git a/tools/block-generator/generator/daemon.go b/tools/block-generator/generator/daemon.go index ac60a40edf..5e15d11c1b 100644 --- a/tools/block-generator/generator/daemon.go +++ b/tools/block-generator/generator/daemon.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 88cf102054..5574370605 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 49cd49d17e..b6bd5bc387 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( @@ -10,6 +26,7 @@ import ( "github.com/stretchr/testify/require" ) +// partitiontest.PartitionTest(t) (partitiontest) func makePrivateGenerator(t *testing.T) *generator { publicGenerator, err := MakeGenerator(GenerationConfig{ NumGenesisAccounts: 10, @@ -22,18 +39,21 @@ func makePrivateGenerator(t *testing.T) *generator { return publicGenerator.(*generator) } +// partitiontest.PartitionTest(t) (partitiontest) func TestPaymentAcctCreate(t *testing.T) { g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentAcctCreateTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts+1)) } +// partitiontest.PartitionTest(t) (partitiontest) func TestPaymentTransfer(t *testing.T) { g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts)) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetXferNoAssetsOverride(t *testing.T) { g := makePrivateGenerator(t) @@ -47,6 +67,7 @@ func TestAssetXferNoAssetsOverride(t *testing.T) { require.Len(t, g.pendingAssets[0].holders, 1) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetXferOneHolderOverride(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -63,6 +84,7 @@ func TestAssetXferOneHolderOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 2) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetCloseCreatorOverride(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -79,6 +101,7 @@ func TestAssetCloseCreatorOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 2) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetOptinEveryAccountOverride(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -111,6 +134,7 @@ func TestAssetOptinEveryAccountOverride(t *testing.T) { require.Len(t, g.assets[0].holders, int(g.numAccounts-1)) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetDestroyWithHoldingsOverride(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -131,6 +155,7 @@ func TestAssetDestroyWithHoldingsOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 1) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetTransfer(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -144,6 +169,7 @@ func TestAssetTransfer(t *testing.T) { require.Greater(t, g.assets[0].holdings[1].balance, uint64(0)) } +// partitiontest.PartitionTest(t) (partitiontest) func TestAssetDestroy(t *testing.T) { g := makePrivateGenerator(t) g.finishRound(0) @@ -157,6 +183,7 @@ func TestAssetDestroy(t *testing.T) { require.Len(t, g.assets, 0) } +// partitiontest.PartitionTest(t) (partitiontest) func TestWriteRoundZero(t *testing.T) { g := makePrivateGenerator(t) var data []byte @@ -167,6 +194,7 @@ func TestWriteRoundZero(t *testing.T) { require.Len(t, block.Block.Payset, 0) } +// partitiontest.PartitionTest(t) (partitiontest) func TestWriteRound(t *testing.T) { g := makePrivateGenerator(t) var data []byte @@ -177,6 +205,7 @@ func TestWriteRound(t *testing.T) { require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) } +// partitiontest.PartitionTest(t) (partitiontest) func TestIndexToAccountAndAccountToIndex(t *testing.T) { for i := uint64(0); i < uint64(100000); i++ { acct := indexToAccount(i) diff --git a/tools/block-generator/generator/make_transactions.go b/tools/block-generator/generator/make_transactions.go index 71e84c9d4c..cd316779ec 100644 --- a/tools/block-generator/generator/make_transactions.go +++ b/tools/block-generator/generator/make_transactions.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 4b8d674e0e..2cb4932865 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 039dd3885f..6fcc0a6c04 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( @@ -10,6 +26,7 @@ import ( "github.com/stretchr/testify/require" ) +// partitiontest.PartitionTest(t) (partitiontest) func TestInitConfigFile(t *testing.T) { config, err := initializeConfigFile("../test_config.yml") require.NoError(t, err) @@ -18,6 +35,7 @@ func TestInitConfigFile(t *testing.T) { require.Equal(t, float32(0.0), config.AssetDestroyFraction) } +// partitiontest.PartitionTest(t) (partitiontest) func TestInitConfigFileNotExist(t *testing.T) { _, err := initializeConfigFile("this_is_not_a_config_file") @@ -26,6 +44,7 @@ func TestInitConfigFileNotExist(t *testing.T) { } } +// partitiontest.PartitionTest(t) (partitiontest) func TestParseRound(t *testing.T) { var testcases = []struct { name string diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index fed2c1ec8c..7c7a3980ce 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 09dbb8b05c..e182838039 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package generator import ( @@ -8,6 +24,7 @@ import ( "github.com/stretchr/testify/require" ) +// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionInternalBadInput(t *testing.T) { weights := []float32{0.10, 0.30} options := []interface{}{"10"} @@ -15,6 +32,7 @@ func TestWeightedSelectionInternalBadInput(t *testing.T) { require.EqualError(t, err, "number of weights must equal number of options: 2 != 1") } +// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionInternal(t *testing.T) { weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} @@ -59,6 +77,7 @@ func TestWeightedSelectionInternal(t *testing.T) { } } +// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelection(t *testing.T) { weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} @@ -74,6 +93,7 @@ func TestWeightedSelection(t *testing.T) { assert.Less(t, selections[options[1]], selections[options[2]]) } +// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionOutOfRange(t *testing.T) { weights := []float32{0.1} options := []interface{}{"1"} diff --git a/tools/block-generator/main.go b/tools/block-generator/main.go index 4a9e1b7c33..eb397f2ef0 100644 --- a/tools/block-generator/main.go +++ b/tools/block-generator/main.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package main import "github.com/algorand/go-algorand/tools/block-generator/core" diff --git a/tools/block-generator/metrics/metrics.go b/tools/block-generator/metrics/metrics.go index 5645023149..6c60fbabc2 100644 --- a/tools/block-generator/metrics/metrics.go +++ b/tools/block-generator/metrics/metrics.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package metrics // Prometheus metrics collected in Conduit. diff --git a/tools/block-generator/runner/metrics_collector.go b/tools/block-generator/runner/metrics_collector.go index 7c89aed02a..327ca9e33b 100644 --- a/tools/block-generator/runner/metrics_collector.go +++ b/tools/block-generator/runner/metrics_collector.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package runner import ( diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 47a2a6a985..9c08282257 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package runner import ( diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index c0bbf7cbe1..94df877660 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package runner import ( diff --git a/tools/block-generator/util/util.go b/tools/block-generator/util/util.go index bead732f66..ad7c73baa5 100644 --- a/tools/block-generator/util/util.go +++ b/tools/block-generator/util/util.go @@ -1,3 +1,19 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + package util import ( From 5615cc3db644ef5039febbbd0e7c0d271b4b97b3 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 30 Mar 2023 14:14:00 -0400 Subject: [PATCH 03/16] fix failing test, lint --- tools/block-generator/generator/generate.go | 24 ++++++++--------- .../generator/generate_test.go | 26 +++++++++---------- .../block-generator/generator/server_test.go | 6 ++--- tools/block-generator/generator/utils_test.go | 8 +++--- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 5574370605..baaf5a239b 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -65,27 +65,27 @@ const ( // GenerationConfig defines the tunable parameters for block generation. type GenerationConfig struct { - Name string `mapstructure:"name"` - NumGenesisAccounts uint64 `mapstructure:"genesis_accounts"` - GenesisAccountInitialBalance uint64 `mapstructure:"genesis_account_balance"` + Name string `yaml:"name"` + NumGenesisAccounts uint64 `yaml:"genesis_accounts"` + GenesisAccountInitialBalance uint64 `yaml:"genesis_account_balance"` // Block generation TxnPerBlock uint64 `mapstructure:"tx_per_block"` // TX Distribution - PaymentTransactionFraction float32 `mapstructure:"tx_pay_fraction"` - AssetTransactionFraction float32 `mapstructure:"tx_asset_fraction"` + PaymentTransactionFraction float32 `yaml:"tx_pay_fraction"` + AssetTransactionFraction float32 `yaml:"tx_asset_fraction"` // Payment configuration - PaymentNewAccountFraction float32 `mapstructure:"pay_acct_create_fraction"` - PaymentFraction float32 `mapstructure:"pay_xfer_fraction"` + PaymentNewAccountFraction float32 `yaml:"pay_acct_create_fraction"` + PaymentFraction float32 `yaml:"pay_xfer_fraction"` // Asset configuration - AssetCreateFraction float32 `mapstructure:"asset_create_fraction"` - AssetDestroyFraction float32 `mapstructure:"asset_destroy_fraction"` - AssetOptinFraction float32 `mapstructure:"asset_optin_fraction"` - AssetCloseFraction float32 `mapstructure:"asset_close_fraction"` - AssetXferFraction float32 `mapstructure:"asset_xfer_fraction"` + AssetCreateFraction float32 `yaml:"asset_create_fraction"` + AssetDestroyFraction float32 `yaml:"asset_destroy_fraction"` + AssetOptinFraction float32 `yaml:"asset_optin_fraction"` + AssetCloseFraction float32 `yaml:"asset_close_fraction"` + AssetXferFraction float32 `yaml:"asset_xfer_fraction"` } func sumIsCloseToOne(numbers ...float32) bool { diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index b6bd5bc387..9d337ce42e 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -26,8 +26,8 @@ import ( "github.com/stretchr/testify/require" ) -// partitiontest.PartitionTest(t) (partitiontest) func makePrivateGenerator(t *testing.T) *generator { + // partitiontest.PartitionTest(t) (partitiontest) publicGenerator, err := MakeGenerator(GenerationConfig{ NumGenesisAccounts: 10, GenesisAccountInitialBalance: 10000000000000000000, @@ -39,22 +39,22 @@ func makePrivateGenerator(t *testing.T) *generator { return publicGenerator.(*generator) } -// partitiontest.PartitionTest(t) (partitiontest) func TestPaymentAcctCreate(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentAcctCreateTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts+1)) } -// partitiontest.PartitionTest(t) (partitiontest) func TestPaymentTransfer(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts)) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetXferNoAssetsOverride(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) // First asset transaction must create. @@ -67,8 +67,8 @@ func TestAssetXferNoAssetsOverride(t *testing.T) { require.Len(t, g.pendingAssets[0].holders, 1) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetXferOneHolderOverride(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -84,8 +84,8 @@ func TestAssetXferOneHolderOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 2) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetCloseCreatorOverride(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -101,8 +101,8 @@ func TestAssetCloseCreatorOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 2) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetOptinEveryAccountOverride(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -134,8 +134,8 @@ func TestAssetOptinEveryAccountOverride(t *testing.T) { require.Len(t, g.assets[0].holders, int(g.numAccounts-1)) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetDestroyWithHoldingsOverride(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -155,8 +155,8 @@ func TestAssetDestroyWithHoldingsOverride(t *testing.T) { require.Len(t, g.assets[0].holders, 1) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetTransfer(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) @@ -169,8 +169,8 @@ func TestAssetTransfer(t *testing.T) { require.Greater(t, g.assets[0].holdings[1].balance, uint64(0)) } -// partitiontest.PartitionTest(t) (partitiontest) func TestAssetDestroy(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -183,8 +183,8 @@ func TestAssetDestroy(t *testing.T) { require.Len(t, g.assets, 0) } -// partitiontest.PartitionTest(t) (partitiontest) func TestWriteRoundZero(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) var data []byte writer := bytes.NewBuffer(data) @@ -194,8 +194,8 @@ func TestWriteRoundZero(t *testing.T) { require.Len(t, block.Block.Payset, 0) } -// partitiontest.PartitionTest(t) (partitiontest) func TestWriteRound(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) g := makePrivateGenerator(t) var data []byte writer := bytes.NewBuffer(data) @@ -205,8 +205,8 @@ func TestWriteRound(t *testing.T) { require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) } -// partitiontest.PartitionTest(t) (partitiontest) func TestIndexToAccountAndAccountToIndex(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) for i := uint64(0); i < uint64(100000); i++ { acct := indexToAccount(i) result := accountToIndex(acct) diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 6fcc0a6c04..3f010fe1f6 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -26,8 +26,8 @@ import ( "github.com/stretchr/testify/require" ) -// partitiontest.PartitionTest(t) (partitiontest) func TestInitConfigFile(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) config, err := initializeConfigFile("../test_config.yml") require.NoError(t, err) require.Equal(t, uint64(10), config.NumGenesisAccounts) @@ -35,8 +35,8 @@ func TestInitConfigFile(t *testing.T) { require.Equal(t, float32(0.0), config.AssetDestroyFraction) } -// partitiontest.PartitionTest(t) (partitiontest) func TestInitConfigFileNotExist(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) _, err := initializeConfigFile("this_is_not_a_config_file") if _, ok := err.(*os.PathError); !ok { @@ -44,8 +44,8 @@ func TestInitConfigFileNotExist(t *testing.T) { } } -// partitiontest.PartitionTest(t) (partitiontest) func TestParseRound(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) var testcases = []struct { name string url string diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index e182838039..78233109f0 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -24,16 +24,16 @@ import ( "github.com/stretchr/testify/require" ) -// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionInternalBadInput(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) weights := []float32{0.10, 0.30} options := []interface{}{"10"} _, err := weightedSelectionInternal(0, weights, options, nil) require.EqualError(t, err, "number of weights must equal number of options: 2 != 1") } -// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionInternal(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} @@ -77,8 +77,8 @@ func TestWeightedSelectionInternal(t *testing.T) { } } -// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelection(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} selections := make(map[interface{}]int) @@ -93,8 +93,8 @@ func TestWeightedSelection(t *testing.T) { assert.Less(t, selections[options[1]], selections[options[2]]) } -// partitiontest.PartitionTest(t) (partitiontest) func TestWeightedSelectionOutOfRange(t *testing.T) { + // partitiontest.PartitionTest(t) (partitiontest) weights := []float32{0.1} options := []interface{}{"1"} defaultOption := "DEFAULT!" From 690eb7cb052948f302abd90ede80236574f3c4f8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 30 Mar 2023 14:29:15 -0400 Subject: [PATCH 04/16] lint fix --- .../generator/generate_test.go | 27 ++++++++++--------- tools/block-generator/generator/server.go | 6 +++-- .../block-generator/generator/server_test.go | 7 ++--- tools/block-generator/generator/utils_test.go | 9 ++++--- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 9d337ce42e..4f25b4d0fb 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -23,11 +23,12 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) func makePrivateGenerator(t *testing.T) *generator { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) publicGenerator, err := MakeGenerator(GenerationConfig{ NumGenesisAccounts: 10, GenesisAccountInitialBalance: 10000000000000000000, @@ -40,21 +41,21 @@ func makePrivateGenerator(t *testing.T) *generator { } func TestPaymentAcctCreate(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentAcctCreateTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts+1)) } func TestPaymentTransfer(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.generatePaymentTxnInternal(paymentTx, 0, 0) require.Len(t, g.balances, int(g.config.NumGenesisAccounts)) } func TestAssetXferNoAssetsOverride(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) // First asset transaction must create. @@ -68,7 +69,7 @@ func TestAssetXferNoAssetsOverride(t *testing.T) { } func TestAssetXferOneHolderOverride(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -85,7 +86,7 @@ func TestAssetXferOneHolderOverride(t *testing.T) { } func TestAssetCloseCreatorOverride(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -102,7 +103,7 @@ func TestAssetCloseCreatorOverride(t *testing.T) { } func TestAssetOptinEveryAccountOverride(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -135,7 +136,7 @@ func TestAssetOptinEveryAccountOverride(t *testing.T) { } func TestAssetDestroyWithHoldingsOverride(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -156,7 +157,7 @@ func TestAssetDestroyWithHoldingsOverride(t *testing.T) { } func TestAssetTransfer(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) @@ -170,7 +171,7 @@ func TestAssetTransfer(t *testing.T) { } func TestAssetDestroy(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) g.finishRound(0) g.generateAssetTxnInternal(assetCreate, 1, 0) @@ -184,7 +185,7 @@ func TestAssetDestroy(t *testing.T) { } func TestWriteRoundZero(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) var data []byte writer := bytes.NewBuffer(data) @@ -195,7 +196,7 @@ func TestWriteRoundZero(t *testing.T) { } func TestWriteRound(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) g := makePrivateGenerator(t) var data []byte writer := bytes.NewBuffer(data) @@ -206,7 +207,7 @@ func TestWriteRound(t *testing.T) { } func TestIndexToAccountAndAccountToIndex(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) for i := uint64(0); i < uint64(100000); i++ { acct := indexToAccount(i) result := accountToIndex(acct) diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 2cb4932865..528cc47282 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -21,6 +21,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/algorand/go-algorand/tools/block-generator/util" "gopkg.in/yaml.v3" @@ -65,8 +66,9 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) return &http.Server{ - Addr: addr, - Handler: mux, + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 3 * time.Second, }, gen } diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 3f010fe1f6..4d2863aeff 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -22,12 +22,13 @@ import ( "strings" "testing" + "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestInitConfigFile(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) config, err := initializeConfigFile("../test_config.yml") require.NoError(t, err) require.Equal(t, uint64(10), config.NumGenesisAccounts) @@ -36,7 +37,7 @@ func TestInitConfigFile(t *testing.T) { } func TestInitConfigFileNotExist(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) _, err := initializeConfigFile("this_is_not_a_config_file") if _, ok := err.(*os.PathError); !ok { @@ -45,7 +46,7 @@ func TestInitConfigFileNotExist(t *testing.T) { } func TestParseRound(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) var testcases = []struct { name string url string diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 78233109f0..7fc289d695 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -20,12 +20,13 @@ import ( "fmt" "testing" + "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestWeightedSelectionInternalBadInput(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) weights := []float32{0.10, 0.30} options := []interface{}{"10"} _, err := weightedSelectionInternal(0, weights, options, nil) @@ -33,7 +34,7 @@ func TestWeightedSelectionInternalBadInput(t *testing.T) { } func TestWeightedSelectionInternal(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} @@ -78,7 +79,7 @@ func TestWeightedSelectionInternal(t *testing.T) { } func TestWeightedSelection(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) weights := []float32{0.10, 0.30, 0.60} options := []interface{}{"10", "30", "60"} selections := make(map[interface{}]int) @@ -94,7 +95,7 @@ func TestWeightedSelection(t *testing.T) { } func TestWeightedSelectionOutOfRange(t *testing.T) { - // partitiontest.PartitionTest(t) (partitiontest) + partitiontest.PartitionTest(t) weights := []float32{0.1} options := []interface{}{"1"} defaultOption := "DEFAULT!" From 5de1ff739fcfc45fbd64868c7281461411192abf Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 30 Mar 2023 15:09:56 -0400 Subject: [PATCH 05/16] update test --- .gitignore | 3 +++ tools/block-generator/generator/generate.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9781154449..39abf02ee5 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,6 @@ index.html # test summary testresults.json + +# block generator binary +tools/block-generator/block-generator diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index baaf5a239b..41fad8736e 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -350,7 +350,7 @@ func (g *generator) finishRound(txnCount uint64) { // WriteBlock generates a block full of new transactions and writes it to the writer. func (g *generator) WriteBlock(output io.Writer, round uint64) error { if round != g.round { - fmt.Printf("Generator only supports sequential block access. Expected %d but received request for %d.", g.round, round) + fmt.Printf("Generator only supports sequential block access. Expected %d but received request for %d.\n", g.round, round) } numTxnForBlock := g.txnForRound(round) From e3a9516295ff562e4307b80c5c50176dee00ff62 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 31 Mar 2023 10:07:43 -0400 Subject: [PATCH 06/16] algofix --- tools/block-generator/runner/run.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 9c08282257..542c259f85 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -26,12 +26,12 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/algorand/go-algorand/tools/block-generator/generator" "github.com/algorand/go-algorand/tools/block-generator/metrics" "github.com/algorand/go-algorand/tools/block-generator/util" + "github.com/algorand/go-deadlock" ) // Args are all the things needed to run a performance test. @@ -90,7 +90,7 @@ func (r *Args) run() error { } // This middleware allows us to lock the block endpoint - var freezeMutex sync.Mutex + var freezeMutex deadlock.Mutex blockMiddleware := func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { freezeMutex.Lock() From 69b79a4970330d364d146daa353619ab48e5a422 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 31 Mar 2023 10:56:07 -0400 Subject: [PATCH 07/16] address reviewdog errors --- tools/block-generator/generator/daemon.go | 5 ++++- tools/block-generator/generator/server.go | 5 ++++- tools/block-generator/runner/run.go | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/block-generator/generator/daemon.go b/tools/block-generator/generator/daemon.go index 5e15d11c1b..36c49cac51 100644 --- a/tools/block-generator/generator/daemon.go +++ b/tools/block-generator/generator/daemon.go @@ -38,7 +38,10 @@ func init() { Run: func(cmd *cobra.Command, args []string) { addr := fmt.Sprintf(":%d", port) srv, _ := MakeServer(configFile, addr) - srv.ListenAndServe() + err := srv.ListenAndServe() + if err != nil { + panic(err) + } }, } diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 528cc47282..10adc04728 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -32,7 +32,10 @@ func initializeConfigFile(configFile string) (config GenerationConfig, err error if err != nil { return } - yaml.Unmarshal(data, &config) + err = yaml.Unmarshal(data, &config) + if err != nil { + return + } return } diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 542c259f85..7d933c557b 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -64,7 +64,10 @@ func Run(args Args) error { return fmt.Errorf("report directory '%s' already exists", args.ReportDirectory) } } - os.Mkdir(args.ReportDirectory, os.ModeDir|os.ModePerm) + err := os.Mkdir(args.ReportDirectory, os.ModeDir|os.ModePerm) + if err != nil { + return err + } defer fmt.Println("Done running tests!") return filepath.Walk(args.Path, func(path string, info os.FileInfo, err error) error { From 1ce79698800553cabc431d21abefd62f57c6f4c8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 31 Mar 2023 11:11:12 -0400 Subject: [PATCH 08/16] address reviewdog errors --- tools/block-generator/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/block-generator/main.go b/tools/block-generator/main.go index eb397f2ef0..6525bc4a6c 100644 --- a/tools/block-generator/main.go +++ b/tools/block-generator/main.go @@ -19,5 +19,8 @@ package main import "github.com/algorand/go-algorand/tools/block-generator/core" func main() { - core.BlockGenerator.Execute() + err := core.BlockGenerator.Execute() + if err != nil { + panic(err) + } } From 147ed1bacd535feda2a8ec717cd886be8b87b334 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 3 Apr 2023 17:13:21 -0400 Subject: [PATCH 09/16] adding new endpoints --- tools/block-generator/generator/generate.go | 63 ++++++++++++++++++--- tools/block-generator/generator/server.go | 14 +++++ tools/block-generator/generator/utils.go | 24 ++++++++ 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 41fad8736e..3c653b494e 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -17,7 +17,7 @@ package generator import ( - "encoding/binary" + "context" "encoding/json" "fmt" "io" @@ -26,6 +26,10 @@ import ( "time" cconfig "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/eval" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/agreement" @@ -133,6 +137,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) { gen.genesisHash[31] = 3 gen.initializeAccounting() + gen.initializeLedger() for _, val := range getTransactionOptions() { switch val { @@ -177,7 +182,9 @@ type Generator interface { WriteBlock(output io.Writer, round uint64) error WriteAccount(output io.Writer, accountString string) error WriteStatus(output io.Writer) error + WriteDeltas(output io.Writer, round uint64) error Accounts() <-chan basics.Address + Stop() } type generator struct { @@ -224,6 +231,10 @@ type generator struct { // Reporting information from transaction type to data reportData Report + + // ledger + ledger *ledger.Ledger + deltas map[uint64]ledgercore.StateDelta } type assetData struct { @@ -411,19 +422,35 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { return err } + // generate deltas for the block + err = g.generateDeltas(cert.Block) + if err != nil { + panic(fmt.Sprintf("failed to generate state deltas: %v\n", err)) + } g.finishRound(numTxnForBlock) return nil } -func indexToAccount(i uint64) (addr basics.Address) { - // Make sure we don't generate a zero address by adding 1 to i - binary.LittleEndian.PutUint64(addr[:], i+1) - return +func (g *generator) generateDeltas(block bookkeeping.Block) error { + delta, err := eval.Eval(context.Background(), g.ledger, block, false, nil, nil) + if err != nil { + return err + } + g.deltas[uint64(block.Round())] = delta + return nil } -func accountToIndex(a basics.Address) (addr uint64) { - // Make sure we don't generate a zero address by adding 1 to i - return binary.LittleEndian.Uint64(a[:]) - 1 +// WriteDeltas generates returns the deltas for payset. +func (g *generator) WriteDeltas(output io.Writer, round uint64) error { + delta, ok := g.deltas[round] + if !ok { + return fmt.Errorf("state deltas for round %d not found", round) + } + err := json.NewEncoder(output).Encode(delta) + if err != nil { + return err + } + return nil } // initializeAccounting creates the genesis accounts. @@ -686,6 +713,26 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S return signTxn(txn), transactions.ApplyData{}, nil } +func (g *generator) initializeLedger() { + bal := bookkeeping.MakeGenesisBalances(convertToGenesisBalances(g.balances), g.feeSink, g.rewardsPool) + block, _ := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash) + l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{ + Block: block, + Accounts: bal.Balances, + GenesisHash: g.genesisHash, + }, cconfig.GetDefaultLocal()) + if err != nil { + fmt.Printf("error initializing ledger: %v\n.", err) + os.Exit(1) + } + g.ledger = l +} + +// Stop cleans up allocated resources. +func (g *generator) Stop() { + g.ledger.Close() +} + func (g *generator) WriteAccount(output io.Writer, accountString string) error { addr, err := basics.UnmarshalChecksumAddress(accountString) if err != nil { diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 10adc04728..268939c85d 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -59,6 +59,7 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B gen, err := MakeGenerator(config) util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile) + defer gen.Stop() mux := http.NewServeMux() mux.HandleFunc("/", help) @@ -67,6 +68,8 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B mux.HandleFunc("/genesis", getGenesisHandler(gen)) mux.HandleFunc("/report", getReportHandler(gen)) mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) + mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) + mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen)) return &http.Server{ Addr: addr, @@ -130,6 +133,17 @@ func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques } } +func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + round, err := parseRound(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + maybeWriteError(w, gen.WriteDeltas(w, round)) + } +} + const blockQueryPrefix = "/v2/blocks/" const blockQueryBlockIdx = len(blockQueryPrefix) const accountsQueryPrefix = "/v2/accounts/" diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index 7c7a3980ce..37d38d7af2 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -17,8 +17,11 @@ package generator import ( + "encoding/binary" "fmt" "math/rand" + + "github.com/algorand/go-algorand/data/basics" ) func weightedSelection(weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) { @@ -43,3 +46,24 @@ func weightedSelectionInternal(selectionNumber float32, weights []float32, optio selection = defaultOption return } + +func indexToAccount(i uint64) (addr basics.Address) { + // Make sure we don't generate a zero address by adding 1 to i + binary.LittleEndian.PutUint64(addr[:], i+1) + return +} + +func accountToIndex(a basics.Address) (addr uint64) { + // Make sure we don't generate a zero address by adding 1 to i + return binary.LittleEndian.Uint64(a[:]) - 1 +} + +func convertToGenesisBalances(balances []uint64) map[basics.Address]basics.AccountData { + genesisBalances := make(map[basics.Address]basics.AccountData) + for i, balance := range balances { + genesisBalances[indexToAccount(uint64(i))] = basics.AccountData{ + MicroAlgos: basics.MicroAlgos{Raw: balance}, + } + } + return genesisBalances +} From 12959eea354ad16a08d70132b781a2e451fd3bf0 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 4 Apr 2023 15:20:36 -0400 Subject: [PATCH 10/16] fixing deltas endpoint --- tools/block-generator/generator/generate.go | 19 ++++-- tools/block-generator/generator/server.go | 67 ++++++++----------- .../block-generator/generator/server_test.go | 65 ------------------ tools/block-generator/runner/run.go | 2 + 4 files changed, 44 insertions(+), 109 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 3c653b494e..e792345f98 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -130,6 +130,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) { rewardsRate: 0, rewardsRecalculationRound: 0, reportData: make(map[TxTypeID]TxData), + deltas: make(map[uint64]ledgercore.StateDelta), } gen.feeSink[31] = 1 @@ -442,11 +443,12 @@ func (g *generator) generateDeltas(block bookkeeping.Block) error { // WriteDeltas generates returns the deltas for payset. func (g *generator) WriteDeltas(output io.Writer, round uint64) error { - delta, ok := g.deltas[round] + deltas, ok := g.deltas[round] if !ok { return fmt.Errorf("state deltas for round %d not found", round) } - err := json.NewEncoder(output).Encode(delta) + // base64 encode deltas + err := json.NewEncoder(output).Encode(protocol.EncodeJSON(deltas)) if err != nil { return err } @@ -714,8 +716,17 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S } func (g *generator) initializeLedger() { - bal := bookkeeping.MakeGenesisBalances(convertToGenesisBalances(g.balances), g.feeSink, g.rewardsPool) - block, _ := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash) + genBal := convertToGenesisBalances(g.balances) + // add rewards pool with min balance + genBal[g.rewardsPool] = basics.AccountData{ + MicroAlgos: basics.MicroAlgos{g.params.MinBalance}, + } + bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool) + block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash) + if err != nil { + fmt.Printf("error making genesis: %v\n.", err) + os.Exit(1) + } l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{ Block: block, Accounts: bal.Balances, diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 268939c85d..f953db4276 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -20,10 +20,12 @@ import ( "fmt" "net/http" "os" + "strconv" "strings" "time" "github.com/algorand/go-algorand/tools/block-generator/util" + "github.com/gorilla/mux" "gopkg.in/yaml.v3" ) @@ -59,21 +61,20 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B gen, err := MakeGenerator(config) util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile) - defer gen.Stop() - - mux := http.NewServeMux() - mux.HandleFunc("/", help) - mux.Handle("/v2/blocks/", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen)))) - mux.HandleFunc("/v2/accounts/", getAccountHandler(gen)) - mux.HandleFunc("/genesis", getGenesisHandler(gen)) - mux.HandleFunc("/report", getReportHandler(gen)) - mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) - mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) - mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen)) + + r := mux.NewRouter() + r.HandleFunc("/", help) + r.Handle("/v2/blocks/{round}", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen)))) + r.HandleFunc("/v2/accounts/", getAccountHandler(gen)) + r.HandleFunc("/genesis", getGenesisHandler(gen)) + r.HandleFunc("/report", getReportHandler(gen)) + r.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) + r.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) + r.HandleFunc("/v2/deltas/{round}", getDeltasHandler(gen)) return &http.Server{ Addr: addr, - Handler: mux, + Handler: r, ReadHeaderTimeout: 3 * time.Second, }, gen } @@ -110,7 +111,13 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // The generator doesn't actually care about the block... - round, err := parseRound(r.URL.Path) + vars := mux.Vars(r) + param, ok := vars["round"] + if !ok { + http.Error(w, "round missing", http.StatusBadRequest) + return + } + round, err := strconv.ParseUint(param, 10, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -135,7 +142,13 @@ func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - round, err := parseRound(r.URL.Path) + vars := mux.Vars(r) + param, ok := vars["round"] + if !ok { + http.Error(w, "round missing", http.StatusBadRequest) + return + } + round, err := strconv.ParseUint(param, 10, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -144,35 +157,9 @@ func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request } } -const blockQueryPrefix = "/v2/blocks/" -const blockQueryBlockIdx = len(blockQueryPrefix) const accountsQueryPrefix = "/v2/accounts/" const accountsQueryAccountIdx = len(accountsQueryPrefix) -func parseRound(path string) (uint64, error) { - if !strings.HasPrefix(path, blockQueryPrefix) { - return 0, fmt.Errorf("not a blocks query: %s", path) - } - - result := uint64(0) - pathlen := len(path) - - if pathlen == blockQueryBlockIdx { - return 0, fmt.Errorf("no block in path") - } - - for i := blockQueryBlockIdx; i < pathlen; i++ { - if path[i] < '0' || path[i] > '9' { - if i == blockQueryBlockIdx { - return 0, fmt.Errorf("no block in path") - } - break - } - result = (uint64(10) * result) + uint64(int(path[i])-'0') - } - return result, nil -} - func parseAccount(path string) (string, error) { if !strings.HasPrefix(path, accountsQueryPrefix) { return "", fmt.Errorf("not a accounts query: %s", path) diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 4d2863aeff..fcfbe3c38e 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -17,13 +17,10 @@ package generator import ( - "fmt" "os" - "strings" "testing" "github.com/algorand/go-algorand/test/partitiontest" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,65 +41,3 @@ func TestInitConfigFileNotExist(t *testing.T) { require.Fail(t, "This should generate a path error") } } - -func TestParseRound(t *testing.T) { - partitiontest.PartitionTest(t) - var testcases = []struct { - name string - url string - expectedRound uint64 - err string - }{ - { - name: "no block", - url: "/v2/blocks/", - expectedRound: 0, - err: "no block in path", - }, - { - name: "no block 2", - url: "/v2/blocks/?nothing", - expectedRound: 0, - err: "no block in path", - }, - { - name: "invalid prefix", - url: "/v2/wrong/prefix/1", - expectedRound: 0, - err: "not a blocks query", - }, - { - name: "normal one digit", - url: fmt.Sprintf("%s1", blockQueryPrefix), - expectedRound: 1, - err: "", - }, - { - name: "normal long number", - url: fmt.Sprintf("%s12345678", blockQueryPrefix), - expectedRound: 12345678, - err: "", - }, - { - name: "with query parameters", - url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), - expectedRound: 1234, - err: "", - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - round, err := parseRound(testcase.url) - if len(testcase.err) == 0 { - msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%d' received error: %v", - testcase.url, testcase.expectedRound, err) - require.NoError(t, err, msg) - assert.Equal(t, testcase.expectedRound, round) - } else { - require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err)) - require.True(t, strings.Contains(err.Error(), testcase.err)) - } - }) - } -} diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 7d933c557b..4675a54a64 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -341,6 +341,8 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha }() return func() error { + // stop generator + generator.Stop() // Shutdown blocks until the server has stopped. if err := server.Shutdown(context.Background()); err != nil { return fmt.Errorf("failed during generator graceful shutdown: %w", err) From c6edc6f323cb66d681b521e9807b4b004a092bd8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 5 Apr 2023 11:46:13 -0400 Subject: [PATCH 11/16] msgp encode delta --- tools/block-generator/generator/generate.go | 32 +++++-------------- .../generator/generate_test.go | 6 +++- tools/block-generator/generator/server.go | 4 +-- tools/block-generator/generator/utils.go | 22 +++++++++++++ 4 files changed, 37 insertions(+), 27 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index e792345f98..f5e2d72481 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -17,7 +17,6 @@ package generator import ( - "context" "encoding/json" "fmt" "io" @@ -27,7 +26,6 @@ import ( cconfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/ledger" - "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -130,7 +128,6 @@ func MakeGenerator(config GenerationConfig) (Generator, error) { rewardsRate: 0, rewardsRecalculationRound: 0, reportData: make(map[TxTypeID]TxData), - deltas: make(map[uint64]ledgercore.StateDelta), } gen.feeSink[31] = 1 @@ -235,7 +232,6 @@ type generator struct { // ledger ledger *ledger.Ledger - deltas map[uint64]ledgercore.StateDelta } type assetData struct { @@ -422,33 +418,21 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return err } - - // generate deltas for the block - err = g.generateDeltas(cert.Block) - if err != nil { - panic(fmt.Sprintf("failed to generate state deltas: %v\n", err)) - } + g.ledger.AddBlock(cert.Block, agreement.Certificate{}) + g.ledger.WaitForCommit(basics.Round(g.round)) g.finishRound(numTxnForBlock) return nil } -func (g *generator) generateDeltas(block bookkeeping.Block) error { - delta, err := eval.Eval(context.Background(), g.ledger, block, false, nil, nil) - if err != nil { - return err - } - g.deltas[uint64(block.Round())] = delta - return nil -} - // WriteDeltas generates returns the deltas for payset. func (g *generator) WriteDeltas(output io.Writer, round uint64) error { - deltas, ok := g.deltas[round] - if !ok { - return fmt.Errorf("state deltas for round %d not found", round) + delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round)) + if err != nil { + return fmt.Errorf("err getting state delta for round %d, %v", round, err) } - // base64 encode deltas - err := json.NewEncoder(output).Encode(protocol.EncodeJSON(deltas)) + // msgp encode deltas + data, err := encode(protocol.CodecHandle, delta) + _, err = output.Write(data) if err != nil { return err } diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 4f25b4d0fb..54478d0365 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" @@ -31,7 +32,7 @@ func makePrivateGenerator(t *testing.T) *generator { partitiontest.PartitionTest(t) publicGenerator, err := MakeGenerator(GenerationConfig{ NumGenesisAccounts: 10, - GenesisAccountInitialBalance: 10000000000000000000, + GenesisAccountInitialBalance: 1000000000000, PaymentTransactionFraction: 1.0, PaymentNewAccountFraction: 1.0, AssetCreateFraction: 1.0, @@ -204,6 +205,9 @@ func TestWriteRound(t *testing.T) { var block rpcs.EncodedBlockCert protocol.Decode(data, &block) require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) + require.Equal(t, basics.Round(1), g.ledger.Latest()) + _, err := g.ledger.GetStateDeltaForRound(1) + require.NoError(t, err) } func TestIndexToAccountAndAccountToIndex(t *testing.T) { diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index f953db4276..469b722618 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -143,12 +143,12 @@ func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - param, ok := vars["round"] + rd, ok := vars["round"] if !ok { http.Error(w, "round missing", http.StatusBadRequest) return } - round, err := strconv.ParseUint(param, 10, 64) + round, err := strconv.ParseUint(rd, 10, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index 37d38d7af2..82afab4e6d 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -22,6 +22,7 @@ import ( "math/rand" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-codec/codec" ) func weightedSelection(weights []float32, options []interface{}, defaultOption interface{}) (selection interface{}, err error) { @@ -67,3 +68,24 @@ func convertToGenesisBalances(balances []uint64) map[basics.Address]basics.Accou } return genesisBalances } + +func encode(handle codec.Handle, obj interface{}) ([]byte, error) { + var output []byte + enc := codec.NewEncoderBytes(&output, handle) + + err := enc.Encode(obj) + if err != nil { + return nil, fmt.Errorf("failed to encode object: %v", err) + } + return output, nil +} + +func decode(handle codec.Handle, data []byte, v interface{}) error { + enc := codec.NewDecoderBytes(data, handle) + + err := enc.Decode(v) + if err != nil { + return fmt.Errorf("failed to decode object: %v", err) + } + return nil +} From e5f69873d81680f19f02d4ce8646c114d6b5a472 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 5 Apr 2023 14:56:42 -0400 Subject: [PATCH 12/16] update test --- tools/block-generator/generator/generate_test.go | 1 + tools/block-generator/generator/utils.go | 10 ---------- tools/block-generator/generator/utils_test.go | 9 +++++++++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 54478d0365..08aa0e62e3 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -205,6 +205,7 @@ func TestWriteRound(t *testing.T) { var block rpcs.EncodedBlockCert protocol.Decode(data, &block) require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock)) + require.NotNil(t, g.ledger) require.Equal(t, basics.Round(1), g.ledger.Latest()) _, err := g.ledger.GetStateDeltaForRound(1) require.NoError(t, err) diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index 82afab4e6d..59ff7f2e06 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -79,13 +79,3 @@ func encode(handle codec.Handle, obj interface{}) ([]byte, error) { } return output, nil } - -func decode(handle codec.Handle, data []byte, v interface{}) error { - enc := codec.NewDecoderBytes(data, handle) - - err := enc.Decode(v) - if err != nil { - return fmt.Errorf("failed to decode object: %v", err) - } - return nil -} diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 7fc289d695..2ab433a18d 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -109,3 +109,12 @@ func TestWeightedSelectionOutOfRange(t *testing.T) { } assert.Fail(t, "Expected an out of range error by this point.") } + +func TestConvertToGenesisBalance(t *testing.T) { + balance := []uint64{100, 200, 300} + genesisBalances := convertToGenesisBalances(balance) + require.Equal(t, 3, len(genesisBalances)) + for i, bal := range balance { + require.Equal(t, bal, genesisBalances[indexToAccount(uint64(i))].MicroAlgos.Raw) + } +} From 4c1830f54bf3797b4edb2bb703f119679b3d44f2 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 5 Apr 2023 15:00:12 -0400 Subject: [PATCH 13/16] update --- tools/block-generator/runner/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 4675a54a64..9604212da1 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -342,7 +342,7 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha return func() error { // stop generator - generator.Stop() + defer generator.Stop() // Shutdown blocks until the server has stopped. if err := server.Shutdown(context.Background()); err != nil { return fmt.Errorf("failed during generator graceful shutdown: %w", err) From f6e09adf4766416d9567acd3b00ae03f6553de6b Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 10 Apr 2023 12:25:30 -0400 Subject: [PATCH 14/16] lint --- tools/block-generator/generator/generate.go | 12 +++++++++--- tools/block-generator/generator/generate_test.go | 9 --------- tools/block-generator/generator/utils_test.go | 10 ++++++++++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index f5e2d72481..f1961587bb 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -230,7 +230,7 @@ type generator struct { // Reporting information from transaction type to data reportData Report - // ledger + // ledger ledger *ledger.Ledger } @@ -418,7 +418,10 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return err } - g.ledger.AddBlock(cert.Block, agreement.Certificate{}) + err = g.ledger.AddBlock(cert.Block, agreement.Certificate{}) + if err != nil { + return err + } g.ledger.WaitForCommit(basics.Round(g.round)) g.finishRound(numTxnForBlock) return nil @@ -432,6 +435,9 @@ func (g *generator) WriteDeltas(output io.Writer, round uint64) error { } // msgp encode deltas data, err := encode(protocol.CodecHandle, delta) + if err != nil { + return err + } _, err = output.Write(data) if err != nil { return err @@ -703,7 +709,7 @@ func (g *generator) initializeLedger() { genBal := convertToGenesisBalances(g.balances) // add rewards pool with min balance genBal[g.rewardsPool] = basics.AccountData{ - MicroAlgos: basics.MicroAlgos{g.params.MinBalance}, + MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance}, } bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool) block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash) diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 08aa0e62e3..98425e7313 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -210,12 +210,3 @@ func TestWriteRound(t *testing.T) { _, err := g.ledger.GetStateDeltaForRound(1) require.NoError(t, err) } - -func TestIndexToAccountAndAccountToIndex(t *testing.T) { - partitiontest.PartitionTest(t) - for i := uint64(0); i < uint64(100000); i++ { - acct := indexToAccount(i) - result := accountToIndex(acct) - require.Equal(t, i, result) - } -} diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 2ab433a18d..8d719cb0f9 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -111,6 +111,7 @@ func TestWeightedSelectionOutOfRange(t *testing.T) { } func TestConvertToGenesisBalance(t *testing.T) { + partitiontest.PartitionTest(t) balance := []uint64{100, 200, 300} genesisBalances := convertToGenesisBalances(balance) require.Equal(t, 3, len(genesisBalances)) @@ -118,3 +119,12 @@ func TestConvertToGenesisBalance(t *testing.T) { require.Equal(t, bal, genesisBalances[indexToAccount(uint64(i))].MicroAlgos.Raw) } } + +func TestIndexToAccountAndAccountToIndex(t *testing.T) { + partitiontest.PartitionTest(t) + for i := uint64(0); i < uint64(100000); i++ { + acct := indexToAccount(i) + result := accountToIndex(acct) + require.Equal(t, i, result) + } +} From 978f4467890e3696e8f31b6c40769254594ebc7c Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 11 Apr 2023 11:15:38 -0400 Subject: [PATCH 15/16] simplify url path parser --- tools/block-generator/generator/server.go | 63 ++++++++----------- .../block-generator/generator/server_test.go | 22 +++++++ tools/block-generator/generator/utils_test.go | 2 + 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 469b722618..e0c3314082 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -25,7 +25,6 @@ import ( "time" "github.com/algorand/go-algorand/tools/block-generator/util" - "github.com/gorilla/mux" "gopkg.in/yaml.v3" ) @@ -62,19 +61,19 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B gen, err := MakeGenerator(config) util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile) - r := mux.NewRouter() - r.HandleFunc("/", help) - r.Handle("/v2/blocks/{round}", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen)))) - r.HandleFunc("/v2/accounts/", getAccountHandler(gen)) - r.HandleFunc("/genesis", getGenesisHandler(gen)) - r.HandleFunc("/report", getReportHandler(gen)) - r.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) - r.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) - r.HandleFunc("/v2/deltas/{round}", getDeltasHandler(gen)) + mux := http.NewServeMux() + mux.HandleFunc("/", help) + mux.Handle("/v2/blocks/", blocksMiddleware(http.HandlerFunc(getBlockHandler(gen)))) + mux.HandleFunc("/v2/accounts/", getAccountHandler(gen)) + mux.HandleFunc("/genesis", getGenesisHandler(gen)) + mux.HandleFunc("/report", getReportHandler(gen)) + mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) + mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {}) + mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen)) return &http.Server{ Addr: addr, - Handler: r, + Handler: mux, ReadHeaderTimeout: 3 * time.Second, }, gen } @@ -111,18 +110,16 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // The generator doesn't actually care about the block... - vars := mux.Vars(r) - param, ok := vars["round"] - if !ok { - http.Error(w, "round missing", http.StatusBadRequest) + s, err := parseURL(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } - round, err := strconv.ParseUint(param, 10, 64) + round, err := strconv.ParseUint(s, 0, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - maybeWriteError(w, gen.WriteBlock(w, round)) } } @@ -130,25 +127,23 @@ func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // The generator doesn't actually care about the block... - account, err := parseAccount(r.URL.Path) + account, err := parseURL(r.URL.Path) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - maybeWriteError(w, gen.WriteAccount(w, account)) } } func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - rd, ok := vars["round"] - if !ok { - http.Error(w, "round missing", http.StatusBadRequest) + s, err := parseURL(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } - round, err := strconv.ParseUint(rd, 10, 64) + round, err := strconv.ParseUint(s, 0, 64) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -157,19 +152,13 @@ func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request } } -const accountsQueryPrefix = "/v2/accounts/" -const accountsQueryAccountIdx = len(accountsQueryPrefix) - -func parseAccount(path string) (string, error) { - if !strings.HasPrefix(path, accountsQueryPrefix) { - return "", fmt.Errorf("not a accounts query: %s", path) +func parseURL(path string) (string, error) { + i := strings.LastIndex(path, "/") + if i == len(path)-1 { + return "", fmt.Errorf("invalid request path, %s", path) } - - pathlen := len(path) - - if pathlen == accountsQueryAccountIdx { - return "", fmt.Errorf("no address in path") + if strings.Contains(path[i+1:], "?") { + return strings.Split(path[i+1:], "?")[0], nil } - - return path[accountsQueryAccountIdx:], nil + return path[i+1:], nil } diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index fcfbe3c38e..1bf4868f41 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -41,3 +41,25 @@ func TestInitConfigFileNotExist(t *testing.T) { require.Fail(t, "This should generate a path error") } } + +func TestParseURL(t *testing.T) { + partitiontest.PartitionTest(t) + _, err := parseURL("http://v2/blocks/") + require.NotNil(t, err) + _, err = parseURL("http://v2/accounts/") + require.NotNil(t, err) + _, err = parseURL("http://v2/deltas/") + require.NotNil(t, err) + + round, err := parseURL("http://v2/blocks/123") + require.Nil(t, err) + require.Equal(t, round, "123") + + addr, err := parseURL("http://v2/accounts/AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4") + require.Nil(t, err) + require.Equal(t, addr, "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4") + + _, err = parseURL("http://v2/deltas/123?Format=msgp") + require.Nil(t, err) + require.Equal(t, round, "123") +} diff --git a/tools/block-generator/generator/utils_test.go b/tools/block-generator/generator/utils_test.go index 8d719cb0f9..fb69a43fee 100644 --- a/tools/block-generator/generator/utils_test.go +++ b/tools/block-generator/generator/utils_test.go @@ -122,6 +122,8 @@ func TestConvertToGenesisBalance(t *testing.T) { func TestIndexToAccountAndAccountToIndex(t *testing.T) { partitiontest.PartitionTest(t) + account := indexToAccount(1) + fmt.Printf("account: %v\n", account) for i := uint64(0); i < uint64(100000); i++ { acct := indexToAccount(i) result := accountToIndex(acct) From 5681dabeed83a39a92ba2500dfd17942d6e68ab4 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 11 Apr 2023 14:46:59 -0400 Subject: [PATCH 16/16] update tests --- .../generator/generate_test.go | 39 ++++++++ .../block-generator/generator/server_test.go | 98 +++++++++++++++---- 2 files changed, 120 insertions(+), 17 deletions(-) diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 98425e7313..78ca9e0323 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -18,6 +18,8 @@ package generator import ( "bytes" + "net/http" + "net/http/httptest" "testing" "github.com/algorand/go-algorand/data/basics" @@ -210,3 +212,40 @@ func TestWriteRound(t *testing.T) { _, err := g.ledger.GetStateDeltaForRound(1) require.NoError(t, err) } + +func TestHandlers(t *testing.T) { + partitiontest.PartitionTest(t) + g := makePrivateGenerator(t) + handler := getBlockHandler(g) + var testcases = []struct { + name string + url string + err string + }{ + { + name: "no block", + url: "/v2/blocks/?nothing", + err: "invalid request path, /", + }, + { + name: "blocks: round must be numeric", + url: "/v2/blocks/round", + err: `strconv.ParseUint: parsing "round": invalid syntax`, + }, + { + name: "deltas: round must be numeric", + url: "/v2/deltas/round", + err: `strconv.ParseUint: parsing "round": invalid syntax`, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + req := httptest.NewRequest("GET", testcase.url, nil) + w := httptest.NewRecorder() + handler(w, req) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Contains(t, w.Body.String(), testcase.err) + }) + } +} diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 1bf4868f41..6e21d77586 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -17,10 +17,13 @@ package generator import ( + "fmt" "os" + "strings" "testing" "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,22 +47,83 @@ func TestInitConfigFileNotExist(t *testing.T) { func TestParseURL(t *testing.T) { partitiontest.PartitionTest(t) - _, err := parseURL("http://v2/blocks/") - require.NotNil(t, err) - _, err = parseURL("http://v2/accounts/") - require.NotNil(t, err) - _, err = parseURL("http://v2/deltas/") - require.NotNil(t, err) - - round, err := parseURL("http://v2/blocks/123") - require.Nil(t, err) - require.Equal(t, round, "123") - - addr, err := parseURL("http://v2/accounts/AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4") - require.Nil(t, err) - require.Equal(t, addr, "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4") + const blockQueryPrefix = "http://v2/blocks/" + const accountQueryPrefix = "http://v2/accounts/" + const deltaQueryPrefix = "http://v2/deltas/" + var testcases = []struct { + name string + url string + expectedRound string + err string + }{ + { + name: "no block", + url: "/v2/blocks/", + expectedRound: "", + err: "invalid request path, /v2/blocks/", + }, + { + name: "normal one digit", + url: fmt.Sprintf("%s1", blockQueryPrefix), + expectedRound: "1", + err: "", + }, + { + name: "normal long number", + url: fmt.Sprintf("%s12345678", blockQueryPrefix), + expectedRound: "12345678", + err: "", + }, + { + name: "with query parameters", + url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), + expectedRound: "1234", + err: "", + }, + { + name: "with query parameters", + url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), + expectedRound: "1234", + err: "", + }, + { + name: "no deltas", + url: "/v2/deltas/", + expectedRound: "", + err: "invalid request path, /v2/deltas/", + }, + { + name: "deltas", + url: fmt.Sprintf("%s123?Format=msgp", deltaQueryPrefix), + expectedRound: "123", + err: "", + }, + { + name: "no account", + url: "/v2/accounts/", + expectedRound: "", + err: "invalid request path, /v2/accounts/", + }, + { + name: "accounts", + url: fmt.Sprintf("%sAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", accountQueryPrefix), + expectedRound: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", + err: "", + }, + } - _, err = parseURL("http://v2/deltas/123?Format=msgp") - require.Nil(t, err) - require.Equal(t, round, "123") + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + round, err := parseURL(testcase.url) + if len(testcase.err) == 0 { + msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%s' received error: %v", + testcase.url, testcase.expectedRound, err) + require.NoError(t, err, msg) + assert.Equal(t, testcase.expectedRound, round) + } else { + require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err)) + require.True(t, strings.Contains(err.Error(), testcase.err)) + } + }) + } }