chore: execution path refactoring (#410)

* chore: first

* chore: add bench in CI

* wip

* wip

* wip

* wip

* wip

* wip

* chore: refactoring of Execute

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* CI

* chore: first

* chore: execute GetVolumes optim

* chore: execute query optim

* chore: improve bench accuracy

* fix: Execute volumes

* fix: CI setup-task token

* chore: add post transactions batch benchmark

* fix: CI

* chore: script compilation caching optimization

* chore: script variables for postings optimization

* fix: to fix: world is not recognized as such in Numscript variables

* chore: cleanup

.github/workflows/benchmarks.yaml

@@ -8,6 +8,8 @@ jobs:
     steps:
       - name: Install task
         uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:

.github/workflows/main.yml

@@ -30,6 +30,21 @@ jobs:
   Test_sqlite:
     uses: formancehq/gh-workflows/.github/workflows/golang-test.yml@main
 
+  Bench:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+          cache: true
+      - name: Run bench
+        run: task install:perf bench:ledger
+
   GoReleaserBuild:
     if: github.event_name != 'release'
     name: 'GoReleaser Build'

.github/workflows/template_sdk-generate.yaml

@@ -29,5 +29,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - name: Install Task
         uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Launch Generate
         run: task sdk:generate VERSION=main -- ${{ matrix.sdk }}

.github/workflows/template_sdk-publish.yaml

@@ -39,6 +39,8 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - name: Install Task
         uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Launch Generate
         run: task sdk:generate VERSION=${{ inputs.VERSION }} -- ${{ matrix.sdk }}
       - uses: stefanzweifel/git-auto-commit-action@v4

.gitignore

@@ -12,3 +12,4 @@ sdk/sdks
 .vscode
 .env
 sqlstorage.test
+ledger.test

Taskfile.yaml

@@ -85,6 +85,20 @@ tasks:
       NUMARY_STORAGE_DRIVER: "postgres"
       NUMARY_STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:ledger@127.0.0.1/ledger"
 
+  bench:ledger:
+    deps: [postgres]
+    cmds:
+      - mkdir -p {{.BENCH_RESULTS_DIR}}
+      - >
+        go test {{.TAGS}} ./pkg/ledger
+        -run=XXX -bench=BenchmarkLedger_{{.RUN}} -benchmem -benchtime=30s -timeout 1h
+        -cpuprofile {{.BENCH_CPU_PROFILE}} -memprofile {{.BENCH_MEM_PROFILE}}
+        | tee {{.BENCH_RESULTS_FILE}}
+      - benchstat {{.BENCH_RESULTS_FILE}}
+    env:
+      NUMARY_STORAGE_DRIVER: "postgres"
+      NUMARY_STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:ledger@127.0.0.1/ledger"
+
   bench:cpu:
     cmds:
       - go tool pprof -http=":" {{.BENCH_CPU_PROFILE}}

go.mod

@@ -61,7 +61,7 @@ require (
 	github.com/docker/docker v20.10.7+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -148,7 +148,7 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
 	golang.org/x/net v0.4.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
 	google.golang.org/grpc v1.51.0 // indirect

go.sum

@@ -78,6 +78,7 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
 github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -130,6 +131,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=
 github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
@@ -832,6 +835,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

pkg/api/controllers/script_controller.go

@@ -36,7 +36,7 @@ func (ctl *ScriptController) PostScript(c *gin.Context) {
 	preview := ok && (strings.ToUpper(value) == "YES" || strings.ToUpper(value) == "TRUE" || value == "1")
 
 	res := ScriptResponse{}
-	txs, err := l.(*ledger.Ledger).Execute(c.Request.Context(), preview, script)
+	execRes, err := l.(*ledger.Ledger).Execute(c.Request.Context(), false, preview, script)
 	if err != nil {
 		var (
 			code    = apierrors.ErrInternal
@@ -60,8 +60,8 @@ func (ctl *ScriptController) PostScript(c *gin.Context) {
 			res.Details = apierrors.EncodeLink(message)
 		}
 	}
-	if len(txs) > 0 {
-		res.Transaction = &txs[0]
+	if len(execRes) > 0 {
+		res.Transaction = &execRes[0]
 	}
 
 	c.JSON(http.StatusOK, res)

pkg/api/controllers/transaction_controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/numary/ledger/pkg/core"
 	"github.com/numary/ledger/pkg/ledger"
 	"github.com/numary/ledger/pkg/storage/sqlstorage"
+	"github.com/pkg/errors"
 )
 
 type TransactionController struct{}
@@ -179,30 +180,38 @@ func (ctl *TransactionController) PostTransaction(c *gin.Context) {
 		return
 	}
 
-	script := core.ScriptData{
-		Script:    payload.Script,
-		Timestamp: payload.Timestamp,
-		Reference: payload.Reference,
-		Metadata:  payload.Metadata,
-	}
+	var res []core.ExpandedTransaction
+	var err error
 
-	if len(payload.Postings) > 0 {
+	if len(payload.Postings) > 0 && payload.Script.Plain != "" ||
+		len(payload.Postings) == 0 && payload.Script.Plain == "" {
+		apierrors.ResponseError(c, ledger.NewValidationError(
+			"invalid payload: should contain either postings or script"))
+		return
+	} else if len(payload.Postings) > 0 {
+		if i, err := payload.Postings.Validate(); err != nil {
+			apierrors.ResponseError(c, ledger.NewValidationError(errors.Wrap(err,
+				fmt.Sprintf("invalid posting %d", i)).Error()))
+			return
+		}
 		txData := core.TransactionData{
 			Postings:  payload.Postings,
 			Timestamp: payload.Timestamp,
 			Reference: payload.Reference,
 			Metadata:  payload.Metadata,
 		}
-		i, err := l.(*ledger.Ledger).ValidatePostings(c.Request.Context(), txData)
-		if err != nil {
-			apierrors.ResponseError(c, ledger.NewTransactionCommitError(i, err))
-			return
+		res, err = l.(*ledger.Ledger).Execute(c.Request.Context(),
+			true, preview, core.TxsToScriptsData(txData)...)
+	} else {
+		script := core.ScriptData{
+			Script:    payload.Script,
+			Timestamp: payload.Timestamp,
+			Reference: payload.Reference,
+			Metadata:  payload.Metadata,
 		}
-		postingsScript := core.TxsToScriptsData(txData)[0]
-		script.Plain = postingsScript.Plain + script.Plain
+		res, err = l.(*ledger.Ledger).Execute(c.Request.Context(),
+			false, preview, script)
 	}
-
-	res, err := l.(*ledger.Ledger).Execute(c.Request.Context(), preview, script)
 	if err != nil {
 		apierrors.ResponseError(c, err)
 		return
@@ -291,13 +300,20 @@ func (ctl *TransactionController) PostTransactionsBatch(c *gin.Context) {
 		return
 	}
 
-	i, err := l.(*ledger.Ledger).ValidatePostings(c.Request.Context(), txs.Transactions...)
-	if err != nil {
-		apierrors.ResponseError(c, ledger.NewTransactionCommitError(i, err))
-		return
+	for i, tx := range txs.Transactions {
+		if len(tx.Postings) == 0 {
+			apierrors.ResponseError(c, ledger.NewValidationError(errors.New(fmt.Sprintf(
+				"invalid transaction %d: no postings", i)).Error()))
+			return
+		}
+		if j, err := tx.Postings.Validate(); err != nil {
+			apierrors.ResponseError(c, ledger.NewValidationError(errors.Wrap(err,
+				fmt.Sprintf("invalid transaction %d: posting %d", i, j)).Error()))
+			return
+		}
 	}
 
-	res, err := l.(*ledger.Ledger).Execute(c.Request.Context(), false,
+	res, err := l.(*ledger.Ledger).Execute(c.Request.Context(), true, false,
 		core.TxsToScriptsData(txs.Transactions...)...)
 	if err != nil {
 		apierrors.ResponseError(c, err)