feat: readd numscript cache (#177)

pkg/ledger/main_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/formancehq/ledger/pkg/ledger/cache"
 	"github.com/formancehq/ledger/pkg/ledger/lock"
+	"github.com/formancehq/ledger/pkg/ledger/numscript"
 	"github.com/formancehq/ledger/pkg/ledger/query"
 	"github.com/formancehq/ledger/pkg/ledger/runner"
 	"github.com/formancehq/ledger/pkg/ledgertesting"
@@ -48,7 +49,10 @@ func runOnLedger(t interface {
 	cacheManager := cache.NewManager(storageDriver)
 	ledgerCache, err := cacheManager.ForLedger(context.Background(), name)
 	require.NoError(t, err)
-	runner, err := runner.New(store, lock.NewInMemory(), ledgerCache, false)
+
+	compiler := numscript.NewCompiler()
+
+	runner, err := runner.New(store, lock.NewInMemory(), ledgerCache, compiler, false)
 	require.NoError(t, err)
 
 	queryWorker := query.NewWorker(query.DefaultWorkerConfig, storageDriver, query.NewNoOpMonitor())

pkg/ledger/numscript/compiler.go

@@ -0,0 +1,46 @@
+package numscript
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+
+	"github.com/bluele/gcache"
+	"github.com/formancehq/ledger/pkg/machine/script/compiler"
+	"github.com/formancehq/ledger/pkg/machine/vm"
+	"github.com/formancehq/ledger/pkg/machine/vm/program"
+	"github.com/pkg/errors"
+)
+
+type Compiler struct {
+	cache gcache.Cache
+}
+
+func (c *Compiler) Compile(ctx context.Context, script string) (*program.Program, error) {
+
+	digest := sha256.New()
+	_, err := digest.Write([]byte(script))
+	if err != nil {
+		return nil, err
+	}
+
+	cacheKey := base64.StdEncoding.EncodeToString(digest.Sum(nil))
+	v, err := c.cache.Get(cacheKey)
+	if err == nil {
+		return v.(*program.Program), nil
+	}
+
+	program, err := compiler.Compile(script)
+	if err != nil {
+		return nil, vm.NewScriptError(vm.ScriptErrorCompilationFailed, errors.Wrap(err, "compiling numscript").Error())
+	}
+	_ = c.cache.Set(cacheKey, program)
+
+	return program, nil
+}
+
+func NewCompiler() *Compiler {
+	return &Compiler{
+		cache: gcache.New(1024).LFU().Build(), // TODO(gfyrag): Make configurable
+	}
+}

pkg/ledger/numscript/compiler_test.go

@@ -0,0 +1,25 @@
+package numscript
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompiler(t *testing.T) {
+
+	script := `send [USD/2 100] (
+	source = @world
+	destination = @bank
+)`
+
+	compiler := NewCompiler()
+	p1, err := compiler.Compile(context.Background(), script)
+	require.NoError(t, err)
+
+	p2, err := compiler.Compile(context.Background(), script)
+	require.NoError(t, err)
+
+	require.Equal(t, p1, p2)
+}

pkg/ledger/runner/manager.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/formancehq/ledger/pkg/ledger/cache"
 	"github.com/formancehq/ledger/pkg/ledger/lock"
+	"github.com/formancehq/ledger/pkg/ledger/numscript"
 	"github.com/formancehq/ledger/pkg/storage"
 )
 
@@ -16,6 +17,7 @@ type Manager struct {
 	lock                lock.Locker
 	allowPastTimestamps bool
 	cacheManager        *cache.Manager
+	compiler            *numscript.Compiler
 	// ledgers store the script runner for each ledger
 	ledgers map[string]*Runner
 }
@@ -36,7 +38,7 @@ func (m *Manager) ForLedger(ctx context.Context, ledger string) (*Runner, error)
 			return nil, err
 		}
 
-		runner, err = New(store, m.lock, cache, m.allowPastTimestamps)
+		runner, err = New(store, m.lock, cache, m.compiler, m.allowPastTimestamps)
 		if err != nil {
 			return nil, err
 		}
@@ -52,5 +54,6 @@ func NewManager(storageDriver storage.Driver, lock lock.Locker, cacheManager *ca
 		allowPastTimestamps: allowPastTimestamps,
 		cacheManager:        cacheManager,
 		ledgers:             map[string]*Runner{},
+		compiler:            numscript.NewCompiler(),
 	}
 }

pkg/ledger/runner/runner.go

@@ -12,8 +12,8 @@ import (
 	"github.com/formancehq/ledger/pkg/ledger/aggregator"
 	"github.com/formancehq/ledger/pkg/ledger/cache"
 	"github.com/formancehq/ledger/pkg/ledger/lock"
+	"github.com/formancehq/ledger/pkg/ledger/numscript"
 	"github.com/formancehq/ledger/pkg/machine"
-	"github.com/formancehq/ledger/pkg/machine/script/compiler"
 	"github.com/formancehq/ledger/pkg/machine/vm"
 	"github.com/formancehq/ledger/pkg/storage"
 	"github.com/pkg/errors"
@@ -35,6 +35,7 @@ type Runner struct {
 	locker lock.Locker
 	// allowPastTimestamps allow to insert transactions in the past
 	allowPastTimestamps bool
+	compiler            *numscript.Compiler
 }
 
 func (r *Runner) GetMoreRecentTransactionDate() core.Time {
@@ -134,12 +135,12 @@ func (r *Runner) acquireInflight(ctx context.Context, script core.RunScript) (*i
 
 func (r *Runner) execute(ctx context.Context, script core.RunScript, dryRun bool) (*core.ExpandedTransaction, map[string]core.Metadata, error) {
 
-	prog, err := compiler.Compile(script.Plain)
+	program, err := r.compiler.Compile(ctx, script.Plain)
 	if err != nil {
 		return nil, nil, vm.NewScriptError(vm.ScriptErrorCompilationFailed, errors.Wrap(err, "compiling numscript").Error())
 	}
 
-	involvedAccounts, err := prog.GetInvolvedAccounts(script.Vars)
+	involvedAccounts, err := program.GetInvolvedAccounts(script.Vars)
 	if err != nil {
 		return nil, nil, vm.NewScriptError(vm.ScriptErrorCompilationFailed, err.Error())
 	}
@@ -150,7 +151,7 @@ func (r *Runner) execute(ctx context.Context, script core.RunScript, dryRun bool
 	}
 	defer unlock(context.Background()) // Use a background context instead of the request one as it could have been cancelled
 
-	result, err := machine.Run(ctx, r.cache, prog, script)
+	result, err := machine.Run(ctx, r.cache, program, script)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -212,7 +213,7 @@ func (r *Runner) releaseInFlightWithTransaction(inFlight *inFlight, transaction
 	}
 }
 
-func New(store storage.LedgerStore, locker lock.Locker, cache *cache.Cache, allowPastTimestamps bool) (*Runner, error) {
+func New(store storage.LedgerStore, locker lock.Locker, cache *cache.Cache, compiler *numscript.Compiler, allowPastTimestamps bool) (*Runner, error) {
 	log, err := store.ReadLastLogWithType(context.Background(), core.NewTransactionLogType, core.RevertedTransactionLogType)
 	if err != nil && err != sql.ErrNoRows {
 		return nil, err
@@ -246,5 +247,6 @@ func New(store storage.LedgerStore, locker lock.Locker, cache *cache.Cache, allo
 		allowPastTimestamps: allowPastTimestamps,
 		nextTxID:            nextTxID,
 		lastTransactionDate: lastTransactionDate,
+		compiler:            compiler,
 	}, nil
 }

pkg/ledger/runner/runner_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/formancehq/ledger/pkg/core"
 	"github.com/formancehq/ledger/pkg/ledger/cache"
 	"github.com/formancehq/ledger/pkg/ledger/lock"
+	"github.com/formancehq/ledger/pkg/ledger/numscript"
 	"github.com/formancehq/ledger/pkg/ledgertesting"
 	"github.com/formancehq/ledger/pkg/machine/vm"
 	"github.com/formancehq/stack/libs/go-libs/pgtesting"
@@ -177,8 +178,10 @@ func TestExecuteScript(t *testing.T) {
 			_, err = store.Initialize(context.Background())
 			require.NoError(t, err)
 
+			compiler := numscript.NewCompiler()
+
 			cache := cache.New(store)
-			runner, err := New(store, lock.NewInMemory(), cache, false)
+			runner, err := New(store, lock.NewInMemory(), cache, compiler, false)
 			require.NoError(t, err)
 
 			if tc.setup != nil {