From fd33470aeb0d89762641daa4488351cb8366b8f2 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Sat, 22 Apr 2023 11:20:46 +0200 Subject: [PATCH] feat: refine benchs scenario --- .gitignore | 2 + Taskfile.yaml | 145 +++++++++++-------------------------- pkg/ledger/ledger_test.go | 29 +++++--- pkg/ledger/query/worker.go | 4 +- 4 files changed, 62 insertions(+), 118 deletions(-) diff --git a/.gitignore b/.gitignore index 6c136da83..27a43f8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ ledger.test antlr-*-complete.jar go.work go.work.sum + +benchs diff --git a/Taskfile.yaml b/Taskfile.yaml index b9ae11fcc..01b70e32a 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -2,60 +2,24 @@ version: "3" -vars: - PKG: "./..." - FAILFAST: "-failfast" - TIMEOUT: "10m" - RUN: ".*" - TAGS: "-tags json1,netgo" - BENCH_TIME: "30s" - BENCH_RESULTS_DIR: "/tmp/benchmarks" - BENCH_RESULTS_FILE: "/tmp/benchmarks/ledger.txt" - BENCH_CPU_PROFILE: "/tmp/benchmarks/ledger.cpu.prof" - BENCH_MEM_PROFILE: "/tmp/benchmarks/ledger.mem.prof" +env: VERBOSE: "false" + BRANCH: + sh: "git rev-parse --abbrev-ref HEAD| cut -d / -f2" + BENCH_RESULTS_DIR: "./benchs" tasks: - default: - cmds: - - task: generate - - task: lint - - task: tests:local - - generate: - cmds: - - go generate {{.PKG}} - lint: cmds: - golangci-lint run --fix {{if eq .VERBOSE "true"}}-v{{end}} tests: - deps: [postgres] - cmds: - - > - go test {{.TAGS}} {{if eq .VERBOSE "true"}}-v{{end}} -coverpkg {{.PKG}} - -coverprofile coverage.out -covermode atomic {{.PKG}} - env: - STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:ledger@127.0.0.1/ledger" - - tests:local: - cmds: - - task: tests:local:postgres - - tests:local:postgres: - deps: [postgres] cmds: - > - go test {{.TAGS}} {{if eq .VERBOSE "true"}}-v{{end}} {{.FAILFAST}} -coverpkg {{.PKG}} - -coverprofile coverage.out -covermode atomic - -run={{.RUN}} -timeout {{.TIMEOUT}} {{.PKG}} | + go test -failfast -coverpkg ./... -coverprofile coverage.out -covermode atomic ./... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' | sed ''/RUN/s//$(printf "\033[34mRUN\033[0m")/'' - - task: print:coverage - env: - STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:ledger@127.0.0.1/ledger" print:coverage: cmds: @@ -67,50 +31,58 @@ tasks: silent: true bench: - deps: [postgres] + internal: true cmds: - mkdir -p {{.BENCH_RESULTS_DIR}} - > - go test {{.TAGS}} {{if eq .VERBOSE "true"}}-v{{end}} {{.PKG}} - -run=XXX -bench={{.RUN}} -benchmem -benchtime={{.BENCH_TIME}} -timeout {{.TIMEOUT}} - -cpuprofile {{.BENCH_CPU_PROFILE}} -memprofile {{.BENCH_MEM_PROFILE}} - | tee {{.BENCH_RESULTS_FILE}} - - benchstat {{.BENCH_RESULTS_FILE}} + go test -run BenchmarkParallelWrites -bench=. + -test.benchmem + -timeout 1h + -memprofile {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-memprofile-{{if eq .ASYNC "true"}}async{{else}}{{end}}.out + -cpuprofile {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-profile-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.out + -benchtime={{if .DURATION}}{{.DURATION}}{{else}}15s{{end}} + -count={{if .COUNT}}{{.COUNT}}{{else}}10{{end}} ./pkg/ledger | tee {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.stats env: - STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:ledger@127.0.0.1/ledger" + ASYNC: "{{.ASYNC}}" + GOMEMLIMIT: 1GiB + GOGC: "1000" # https://dave.cheney.net/tag/gogc - bench:cpu: + bench:async: cmds: - - go tool pprof -http=":" {{.BENCH_CPU_PROFILE}} + - task: bench + vars: + ASYNC: true - bench:mem: + bench:sync: cmds: - - go tool pprof -http=":" {{.BENCH_MEM_PROFILE}} + - task: bench - install: - deps: - - install:golangci-lint - - install:perf + bench:both: + cmds: + - task: bench:sync + - task: bench:async - install:golangci-lint: + bench:diff: + internal: true cmds: - - > - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | - sh -s -- -b $(go env GOPATH)/bin latest - - golangci-lint --version + - > + benchstat {{.BENCH_RESULTS_DIR}}/{{if .SOURCE}}{{.SOURCE}}{{else}}v2{{end}}-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.stats + {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.stats - install:perf: - - go install golang.org/x/perf/cmd/...@latest + bench:diff:sync: + cmds: + - task: bench:diff - postgres: + bench:diff:async: cmds: - - docker compose up -d postgres + - task: bench:diff + vars: + ASYNC: "true" - clean: + bench:diff:both: cmds: - - go clean - - rm -f {{.SERVICE}} coverage.out coverage.html - - docker compose down -v + - task: bench:diff:sync + - task: bench:diff:async sdk:template: desc: Extract templates @@ -142,22 +114,6 @@ tasks: --git-repo-id=formance-sdk-{{.CLI_ARGS}} -p packageVersion={{.VERSION}} - sdk:test: - desc: Test client code - dir: ./sdk - preconditions: - - sh: '[ "{{.CLI_ARGS}}" != "" ]' - msg: Please specify generator as first cli arg (ie "task test -- go") - - sh: "[[ -e sdks/{{.CLI_ARGS}}/Taskfile.yml ]]" - msg: "Not Taskfile found. You have to create a taskfile in ./sdks/{{.CLI_ARGS}}/ with a 'test' task" - vars: - RANDOM: - sh: "echo $$" - cmds: - - | - cd ./sdks/{{.CLI_ARGS}} - task test - goreleaser:test:pkg: desc: Test a package cmds: @@ -194,22 +150,3 @@ tasks: Platform: "arm64" Image: ubuntu Cmd: "{{.dpkg}} *_linux_arm64.deb" - - run: - cmds: - - docker compose up -d --remove-orphans - - stop: - cmds: - - docker compose stop - - ps: - cmds: - - docker compose ps - - rm: - deps: [stop] - cmds: - - docker compose rm -f - - docker volume prune -f - - rm -f {{.BENCH_CPU_PROFILE}} {{.BENCH_MEM_PROFILE}} diff --git a/pkg/ledger/ledger_test.go b/pkg/ledger/ledger_test.go index 5a9f6641f..19ce31f18 100644 --- a/pkg/ledger/ledger_test.go +++ b/pkg/ledger/ledger_test.go @@ -2,9 +2,11 @@ package ledger import ( "context" + "fmt" + "math/rand" + "os" "testing" - "github.com/alitto/pond" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/ledger/command" "github.com/formancehq/ledger/pkg/storage/sqlstorage/sqlstoragetesting" @@ -22,21 +24,24 @@ func BenchmarkParallelWrites(b *testing.B) { ledger, err := resolver.GetLedger(context.Background(), uuid.NewString()) require.NoError(b, err) - worker := pond.New(1000, 1000) + r := rand.New(rand.NewSource(0)) + + b.SetParallelism(1000) b.ResetTimer() - for i := 0; i < b.N; i++ { - worker.Submit(func() { - _, err := ledger.CreateTransaction(context.Background(), command.Parameters{}, core.RunScript{ + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := ledger.CreateTransaction(context.Background(), command.Parameters{ + Async: os.Getenv("ASYNC") == "true", + }, core.RunScript{ Script: core.Script{ - Plain: `send [USD/2 100] ( - source = @world - destination = @bank - )`, + Plain: fmt.Sprintf(`send [USD/2 100] ( + source = @world + destination = @accounts:%d + )`, r.Int()%100), }, }) require.NoError(b, err) - }) - } - worker.StopAndWait() + } + }) b.StopTimer() } diff --git a/pkg/ledger/query/worker.go b/pkg/ledger/query/worker.go index 51d9b2a1c..811d20dfc 100644 --- a/pkg/ledger/query/worker.go +++ b/pkg/ledger/query/worker.go @@ -101,7 +101,7 @@ l: return nil case w.jobs <- w.pending: - w.pending = make([]*core.LogHolder, 0) + w.pending = make([]*core.LogHolder, 0, 1024) continue l } } @@ -393,7 +393,7 @@ func NewWorker( metricsRegistry metrics.PerLedgerMetricsRegistry, ) *Worker { return &Worker{ - pending: make([]*core.LogHolder, 0), + pending: make([]*core.LogHolder, 0, 1024), jobs: make(chan []*core.LogHolder), releasedJob: make(chan struct{}, 1), writeChannel: make(chan *core.LogHolder, config.ChanSize),