feat: add idempotency at log level

openapi.yaml

@@ -241,6 +241,23 @@ paths:
           schema:
             type: string
             example: users:001
+        - name: dryRun
+          in: query
+          description: Set the dry run mode. Dry run mode doesn't add the logs to the database or publish a message to the message broker.
+          schema:
+            type: boolean
+            example: true
+        - name: async
+          in: query
+          description: Set async mode.
+          schema:
+            type: boolean
+            example: true
+        - name: Idempotency-Key
+          in: header
+          description: Use an idempotency key
+          schema:
+            type: string
       requestBody:
         description: metadata
         content:
@@ -488,6 +505,17 @@ paths:
           schema:
             type: boolean
             example: true
+        - name: async
+          in: query
+          description: Set async mode.
+          schema:
+            type: boolean
+            example: true
+        - name: Idempotency-Key
+          in: header
+          description: Use an idempotency key
+          schema:
+            type: string
       requestBody:
         required: true
         description: >
@@ -572,6 +600,23 @@ paths:
             format: int64
             minimum: 0
             example: 1234
+        - name: dryRun
+          in: query
+          description: Set the dryRun mode. Dry run mode doesn't add the logs to the database or publish a message to the message broker.
+          schema:
+            type: boolean
+            example: true
+        - name: async
+          in: query
+          description: Set async mode.
+          schema:
+            type: boolean
+            example: true
+        - name: Idempotency-Key
+          in: header
+          description: Use an idempotency key
+          schema:
+            type: string
       requestBody:
         description: metadata
         content:

pkg/api/controllers/account_controller.go

@@ -125,7 +125,7 @@ func PostAccountMetadata(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err := l.SaveMeta(r.Context(), core.MetaTargetTypeAccount, chi.URLParam(r, "address"), m, false)
+	err := l.SaveMeta(r.Context(), getCommandParameters(r), core.MetaTargetTypeAccount, chi.URLParam(r, "address"), m)
 	if err != nil {
 		apierrors.ResponseError(w, r, err)
 		return

pkg/api/controllers/account_controller_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/formancehq/ledger/pkg/api/controllers"
 	"github.com/formancehq/ledger/pkg/api/routes"
 	"github.com/formancehq/ledger/pkg/core"
+	"github.com/formancehq/ledger/pkg/ledger/command"
 	"github.com/formancehq/ledger/pkg/opentelemetry/metrics"
 	"github.com/formancehq/ledger/pkg/storage"
 	sharedapi "github.com/formancehq/stack/libs/go-libs/api"
@@ -276,7 +277,7 @@ func TestPostAccountMetadata(t *testing.T) {
 			backend, mock := newTestingBackend(t)
 			if testCase.expectStatusCode == http.StatusNoContent {
 				mock.EXPECT().
-					SaveMeta(gomock.Any(), core.MetaTargetTypeAccount, testCase.account, testCase.body, false).
+					SaveMeta(gomock.Any(), command.Parameters{}, core.MetaTargetTypeAccount, testCase.account, testCase.body).
 					Return(nil)
 			}
 

pkg/api/controllers/api.go

@@ -15,7 +15,6 @@ import (
 
 type Ledger interface {
 	GetAccount(ctx context.Context, param string) (*core.AccountWithVolumes, error)
-	SaveMeta(ctx context.Context, targetType string, targetID any, m metadata.Metadata, async bool) error
 	GetAccounts(ctx context.Context, query storage.AccountsQuery) (*api.Cursor[core.Account], error)
 	CountAccounts(ctx context.Context, query storage.AccountsQuery) (uint64, error)
 	GetBalancesAggregated(ctx context.Context, q storage.BalancesQuery) (core.AssetsBalances, error)
@@ -25,9 +24,11 @@ type Ledger interface {
 	GetLogs(ctx context.Context, query storage.LogsQuery) (*api.Cursor[core.Log], error)
 	CountTransactions(ctx context.Context, query storage.TransactionsQuery) (uint64, error)
 	GetTransactions(ctx context.Context, query storage.TransactionsQuery) (*api.Cursor[core.ExpandedTransaction], error)
-	CreateTransaction(ctx context.Context, parameters command.Parameters, data core.RunScript) (*core.Transaction, error)
 	GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error)
-	RevertTransaction(ctx context.Context, id uint64, async bool) (*core.Transaction, error)
+
+	CreateTransaction(ctx context.Context, parameters command.Parameters, data core.RunScript) (*core.Transaction, error)
+	RevertTransaction(ctx context.Context, parameters command.Parameters, id uint64) (*core.Transaction, error)
+	SaveMeta(ctx context.Context, parameters command.Parameters, targetType string, targetID any, m metadata.Metadata) error
 }
 
 type Backend interface {

pkg/api/controllers/query.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"net/http"
 	"strconv"
+	"strings"
 
 	"github.com/formancehq/ledger/pkg/ledger/command"
 	"github.com/formancehq/ledger/pkg/storage"
@@ -63,3 +64,19 @@ func getBalanceOperator(w http.ResponseWriter, r *http.Request) (storage.Balance
 
 	return balanceOperator, nil
 }
+
+func getCommandParameters(r *http.Request) command.Parameters {
+	dryRunAsString := r.URL.Query().Get("dryRun")
+	dryRun := strings.ToUpper(dryRunAsString) == "YES" || strings.ToUpper(dryRunAsString) == "TRUE" || dryRunAsString == "1"
+
+	asyncAsString := r.URL.Query().Get("async")
+	async := strings.ToUpper(asyncAsString) == "YES" || strings.ToUpper(asyncAsString) == "TRUE" || asyncAsString == "1"
+
+	idempotencyKey := r.Header.Get("Idempotency-Key")
+
+	return command.Parameters{
+		DryRun:         dryRun,
+		Async:          async,
+		IdempotencyKey: idempotencyKey,
+	}
+}

pkg/api/controllers/transaction_controller.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/formancehq/ledger/pkg/api/apierrors"
 	"github.com/formancehq/ledger/pkg/core"
@@ -152,9 +151,6 @@ type PostTransactionRequest struct {
 func PostTransaction(w http.ResponseWriter, r *http.Request) {
 	l := LedgerFromContext(r.Context())
 
-	value := r.URL.Query().Get("dryRun")
-	dryRun := strings.ToUpper(value) == "YES" || strings.ToUpper(value) == "TRUE" || value == "1"
-
 	payload := PostTransactionRequest{}
 	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
 		apierrors.ResponseError(w, r,
@@ -181,10 +177,7 @@ func PostTransaction(w http.ResponseWriter, r *http.Request) {
 			Metadata:  payload.Metadata,
 		}
 
-		res, err := l.CreateTransaction(r.Context(), command.Parameters{
-			DryRun: dryRun,
-			Async:  false,
-		}, core.TxToScriptData(txData))
+		res, err := l.CreateTransaction(r.Context(), getCommandParameters(r), core.TxToScriptData(txData))
 		if err != nil {
 			apierrors.ResponseError(w, r, err)
 			return
@@ -201,10 +194,7 @@ func PostTransaction(w http.ResponseWriter, r *http.Request) {
 		Metadata:  payload.Metadata,
 	}
 
-	res, err := l.CreateTransaction(r.Context(), command.Parameters{
-		DryRun: dryRun,
-		Async:  false,
-	}, script)
+	res, err := l.CreateTransaction(r.Context(), getCommandParameters(r), script)
 	if err != nil {
 		apierrors.ResponseError(w, r, err)
 		return
@@ -242,7 +232,7 @@ func RevertTransaction(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	tx, err := l.RevertTransaction(r.Context(), txId, false)
+	tx, err := l.RevertTransaction(r.Context(), getCommandParameters(r), txId)
 	if err != nil {
 		apierrors.ResponseError(w, r, err)
 		return
@@ -268,7 +258,7 @@ func PostTransactionMetadata(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := l.SaveMeta(r.Context(), core.MetaTargetTypeTransaction, txId, m, false); err != nil {
+	if err := l.SaveMeta(r.Context(), getCommandParameters(r), core.MetaTargetTypeTransaction, txId, m); err != nil {
 		apierrors.ResponseError(w, r, err)
 		return
 	}

pkg/api/controllers/transaction_controller_test.go

@@ -215,7 +215,7 @@ func TestPostTransactionMetadata(t *testing.T) {
 			backend, mock := newTestingBackend(t)
 			if testCase.expectStatusCode == http.StatusNoContent {
 				mock.EXPECT().
-					SaveMeta(gomock.Any(), core.MetaTargetTypeTransaction, uint64(0), testCase.body, false).
+					SaveMeta(gomock.Any(), command.Parameters{}, core.MetaTargetTypeTransaction, uint64(0), testCase.body).
 					Return(nil)
 			}
 
@@ -616,7 +616,7 @@ func TestRevertTransaction(t *testing.T) {
 	backend, mockLedger := newTestingBackend(t)
 	mockLedger.
 		EXPECT().
-		RevertTransaction(gomock.Any(), uint64(0), false).
+		RevertTransaction(gomock.Any(), command.Parameters{}, uint64(0)).
 		Return(&expectedTx, nil)
 
 	router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry())

pkg/core/log.go

@@ -18,6 +18,7 @@ import (
 type LogType int16
 
 const (
+	// TODO(gfyrag): Create dedicated log type for account and metadata
 	SetMetadataLogType         LogType = iota // "SET_METADATA"
 	NewTransactionLogType                     // "NEW_TRANSACTION"
 	RevertedTransactionLogType                // "REVERTED_TRANSACTION"
@@ -68,11 +69,12 @@ type hashable interface {
 
 // TODO(polo): create Log struct and extended Log struct
 type Log struct {
-	ID   uint64   `json:"id"`
-	Type LogType  `json:"type"`
-	Data hashable `json:"data"`
-	Hash []byte   `json:"hash"`
-	Date Time     `json:"date"`
+	ID             uint64   `json:"id"`
+	Type           LogType  `json:"type"`
+	Data           hashable `json:"data"`
+	Hash           []byte   `json:"hash"`
+	Date           Time     `json:"date"`
+	IdempotencyKey string   `json:"idempotencyKey"`
 }
 
 func (l *Log) UnmarshalJSON(data []byte) error {
@@ -106,6 +108,7 @@ func (l *Log) ComputeHash(previous *Log) {
 		buf.writeUInt64(l.ID)
 		buf.writeUInt16(uint16(l.Type))
 		buf.writeUInt64(uint64(l.Date.UnixNano()))
+		buf.writeString(l.IdempotencyKey)
 		l.Data.hashString(buf)
 	}
 
@@ -128,6 +131,11 @@ func (l Log) WithDate(date Time) Log {
 	return l
 }
 
+func (l Log) WithIdempotencyKey(key string) Log {
+	l.IdempotencyKey = key
+	return l
+}
+
 func (l Log) WithID(id uint64) Log {
 	l.ID = id
 	return l

pkg/core/numscript.go

@@ -115,8 +115,8 @@ func TxToScriptData(txData TransactionData) RunScript {
 			Vars:  vars,
 		},
 		Timestamp: txData.Timestamp,
-		Reference: txData.Reference,
 		Metadata:  txData.Metadata,
+		Reference: txData.Reference,
 	}
 }
 

pkg/core/script.go

@@ -9,8 +9,8 @@ import (
 type RunScript struct {
 	Script
 	Timestamp Time              `json:"timestamp"`
-	Reference string            `json:"reference"`
 	Metadata  metadata.Metadata `json:"metadata"`
+	Reference string            `json:"reference"`
 }
 
 type Script struct {

pkg/core/transaction.go

@@ -10,9 +10,9 @@ type Transactions struct {
 
 type TransactionData struct {
 	Postings  Postings          `json:"postings"`
-	Reference string            `json:"reference"`
 	Metadata  metadata.Metadata `json:"metadata"`
 	Timestamp Time              `json:"timestamp"`
+	Reference string            `json:"reference"`
 }
 
 func (d TransactionData) WithPostings(postings ...Posting) TransactionData {
@@ -31,14 +31,9 @@ func (t *TransactionData) Reverse() TransactionData {
 	copy(postings, t.Postings)
 	postings.Reverse()
 
-	ret := TransactionData{
+	return TransactionData{
 		Postings: postings,
 	}
-	//TODO(gfyra): Do we keep this for v2?
-	if t.Reference != "" {
-		ret.Reference = "revert_" + t.Reference
-	}
-	return ret
 }
 
 func (d TransactionData) hashString(buf *buffer) {
@@ -65,8 +60,8 @@ func (t Transaction) WithPostings(postings ...Posting) Transaction {
 	return t
 }
 
-func (t Transaction) WithReference(reference string) Transaction {
-	t.Reference = reference
+func (t Transaction) WithReference(ref string) Transaction {
+	t.Reference = ref
 	return t
 }
 

pkg/core/transaction_test.go

@@ -35,7 +35,6 @@ func TestReverseTransaction(t *testing.T) {
 					Asset:       "COIN",
 				},
 			},
-			Reference: "revert_foo",
 		}
 		require.Equal(t, expected, tx.Reverse())
 	})
@@ -78,7 +77,6 @@ func TestReverseTransaction(t *testing.T) {
 					Asset:       "COIN",
 				},
 			},
-			Reference: "revert_foo",
 		}
 		require.Equal(t, expected, tx.Reverse())
 	})
@@ -133,7 +131,6 @@ func TestReverseTransaction(t *testing.T) {
 					Asset:       "COIN",
 				},
 			},
-			Reference: "revert_foo",
 		}
 		require.Equal(t, expected, tx.Reverse())
 	})