From d1469e2cbb91721674b2c9e970521fd09c167413 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 7 Oct 2021 15:53:40 +0200 Subject: [PATCH 1/5] Run benchmarks on circle CI --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4744bed7b2..d1c30fb2ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,20 @@ jobs: - store_artifacts: path: /tmp/logs + benchmark: + executor: golang + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - go-mod-v1-{{ checksum "go.sum" }} + - run: + name: Run benchmarks + command: | + cd ./x/wasm/keeper + go test -bench . + upload-coverage: executor: golang steps: @@ -178,3 +192,6 @@ workflows: - upload-coverage: requires: - test-cover + - benchmark: + requires: + - test-cover From 18970facf03cc896987be9331bafd532fb62556b Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 7 Oct 2021 16:02:53 +0200 Subject: [PATCH 2/5] Add benchmark for secp256k1 verification --- x/wasm/keeper/bench_test.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/x/wasm/keeper/bench_test.go b/x/wasm/keeper/bench_test.go index 21b16649eb..aae994a73a 100644 --- a/x/wasm/keeper/bench_test.go +++ b/x/wasm/keeper/bench_test.go @@ -1,11 +1,14 @@ package keeper import ( - "github.com/CosmWasm/wasmd/x/wasm/types" + "testing" + + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" "github.com/stretchr/testify/require" "github.com/syndtr/goleveldb/leveldb/opt" dbm "github.com/tendermint/tm-db" - "testing" + + "github.com/CosmWasm/wasmd/x/wasm/types" ) func BenchmarkExecution(b *testing.B) { @@ -53,3 +56,23 @@ func BenchmarkExecution(b *testing.B) { }) } } + +// BenchmarkVerification benchmarks the given verification algorithm using +// the provided privkey on a constant message. +// Copied from https://github.com/cosmos/cosmos-sdk/blob/90e9370bd80d9a3d41f7203ddb71166865561569/crypto/keys/internal/benchmarking/bench.go#L48-L62 +// And thus under the GO license (BSD style) +func BenchmarkSecpVerification(b *testing.B) { + priv := secp256k1.GenPrivKey() + pub := priv.PubKey() + + // use a short message, so this time doesn't get dominated by hashing. + message := []byte("Hello, world!") + signature, err := priv.Sign(message) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + pub.VerifySignature(message, signature) + } +} From 3fd1ce5eb2c8a5d6fe3d4a9f332d9927d937d020 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 7 Oct 2021 16:08:06 +0200 Subject: [PATCH 3/5] Add compilation benchmark --- .circleci/config.yml | 2 +- x/wasm/keeper/bench_test.go | 106 +++++++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 34 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1c30fb2ec..b7d3ce696b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,7 @@ jobs: benchmark: executor: golang - parallelism: 1 + parallelism: 2 steps: - checkout - restore_cache: diff --git a/x/wasm/keeper/bench_test.go b/x/wasm/keeper/bench_test.go index aae994a73a..f1307f6260 100644 --- a/x/wasm/keeper/bench_test.go +++ b/x/wasm/keeper/bench_test.go @@ -1,18 +1,42 @@ package keeper import ( + "fmt" + "io/ioutil" "testing" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" "github.com/stretchr/testify/require" - "github.com/syndtr/goleveldb/leveldb/opt" dbm "github.com/tendermint/tm-db" "github.com/CosmWasm/wasmd/x/wasm/types" ) -func BenchmarkExecution(b *testing.B) { +// BenchmarkVerification benchmarks secp256k1 verification which is 1000 gas based on cpu time. +// +// Just this function is copied from +// https://github.com/cosmos/cosmos-sdk/blob/90e9370bd80d9a3d41f7203ddb71166865561569/crypto/keys/internal/benchmarking/bench.go#L48-L62 +// And thus under the GO license (BSD style) +func BenchmarkGasNormalization(b *testing.B) { + priv := secp256k1.GenPrivKey() + pub := priv.PubKey() + + // use a short message, so this time doesn't get dominated by hashing. + message := []byte("Hello, world!") + signature, err := priv.Sign(message) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + pub.VerifySignature(message, signature) + } +} +// By comparing the timing for queries on pinned vs unpinned, the difference gives us the overhead of +// instantiating an unpinned contract. That value can be used to determine a reasonable gas price +// for the InstantiationCost +func BenchmarkInstantiationOverhead(b *testing.B) { specs := map[string]struct { pinned bool db func() dbm.DB @@ -24,21 +48,21 @@ func BenchmarkExecution(b *testing.B) { db: func() dbm.DB { return dbm.NewMemDB() }, pinned: true, }, - "unpinned, level db": { - db: func() dbm.DB { - levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) - require.NoError(b, err) - return levelDB - }, - }, - "pinned, level db": { - db: func() dbm.DB { - levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) - require.NoError(b, err) - return levelDB - }, - pinned: true, - }, + //"unpinned, level db": { + // db: func() dbm.DB { + // levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) + // require.NoError(b, err) + // return levelDB + // }, + //}, + //"pinned, level db": { + // db: func() dbm.DB { + // levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) + // require.NoError(b, err) + // return levelDB + // }, + // pinned: true, + //}, } for name, spec := range specs { b.Run(name, func(b *testing.B) { @@ -57,22 +81,38 @@ func BenchmarkExecution(b *testing.B) { } } -// BenchmarkVerification benchmarks the given verification algorithm using -// the provided privkey on a constant message. -// Copied from https://github.com/cosmos/cosmos-sdk/blob/90e9370bd80d9a3d41f7203ddb71166865561569/crypto/keys/internal/benchmarking/bench.go#L48-L62 -// And thus under the GO license (BSD style) -func BenchmarkSecpVerification(b *testing.B) { - priv := secp256k1.GenPrivKey() - pub := priv.PubKey() - - // use a short message, so this time doesn't get dominated by hashing. - message := []byte("Hello, world!") - signature, err := priv.Sign(message) - if err != nil { - b.Fatal(err) +// Calculate the time it takes to compile some wasm code the first time. +// This will help us adjust pricing for UploadCode +func BenchmarkCompilation(b *testing.B) { + specs := map[string]struct { + wasmFile string + }{ + "hackatom": { + wasmFile: "./testdata/hackatom.wasm", + }, + "burner": { + wasmFile: "./testdata/burner.wasm", + }, + "ibc_reflect": { + wasmFile: "./testdata/ibc_reflect.wasm", + }, } - b.ResetTimer() - for i := 0; i < b.N; i++ { - pub.VerifySignature(message, signature) + + for name, spec := range specs { + b.Run(name, func(b *testing.B) { + wasmConfig := types.WasmConfig{MemoryCacheSize: 0} + db := dbm.NewMemDB() + ctx, keepers := createTestInput(b, false, SupportedFeatures, wasmConfig, db) + + // print out code size for comparisons + code, err := ioutil.ReadFile(spec.wasmFile) + require.NoError(b, err) + fmt.Printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b(size: %d) ", len(code)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = StoreExampleContract(b, ctx, keepers, spec.wasmFile) + } + }) } } From 5ed122da0c05f48316ed4551f0155d4642e6241a Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 7 Oct 2021 16:29:25 +0200 Subject: [PATCH 4/5] Move parallelism back to 1 for benchmarks --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7d3ce696b..d1c30fb2ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,7 @@ jobs: benchmark: executor: golang - parallelism: 2 + parallelism: 1 steps: - checkout - restore_cache: From 7d891f2775791e4d6910374a953f6f0645b9fcfe Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 8 Oct 2021 12:04:57 +0200 Subject: [PATCH 5/5] Review comments --- x/wasm/keeper/bench_test.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/x/wasm/keeper/bench_test.go b/x/wasm/keeper/bench_test.go index f1307f6260..ca5489d5bb 100644 --- a/x/wasm/keeper/bench_test.go +++ b/x/wasm/keeper/bench_test.go @@ -1,7 +1,6 @@ package keeper import ( - "fmt" "io/ioutil" "testing" @@ -48,21 +47,6 @@ func BenchmarkInstantiationOverhead(b *testing.B) { db: func() dbm.DB { return dbm.NewMemDB() }, pinned: true, }, - //"unpinned, level db": { - // db: func() dbm.DB { - // levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) - // require.NoError(b, err) - // return levelDB - // }, - //}, - //"pinned, level db": { - // db: func() dbm.DB { - // levelDB, err := dbm.NewGoLevelDBWithOpts("testing", b.TempDir(), &opt.Options{BlockCacher: opt.NoCacher}) - // require.NoError(b, err) - // return levelDB - // }, - // pinned: true, - //}, } for name, spec := range specs { b.Run(name, func(b *testing.B) { @@ -107,7 +91,7 @@ func BenchmarkCompilation(b *testing.B) { // print out code size for comparisons code, err := ioutil.ReadFile(spec.wasmFile) require.NoError(b, err) - fmt.Printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b(size: %d) ", len(code)) + b.Logf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b(size: %d) ", len(code)) b.ResetTimer() for i := 0; i < b.N; i++ {