feat: move long migrations in dedicated dir

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 volumes:
   postgres: {}
 

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bluele/gcache v0.0.2
 	github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3
-	github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7
+	github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289
 	github.com/formancehq/ledger/pkg/client v0.0.0-00010101000000-000000000000
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1

go.sum

@@ -106,6 +106,10 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7 h1:OZz4N9nIj814aIgpqIvojndtae+N9Vqj5MJgKPIzJ5Y=
 github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241030160027-898dbd1a42af h1:7POEnA2uHO+a8HNO+LGCuSVpBolflAFcNR0N1BhEuRA=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241030160027-898dbd1a42af/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289 h1:XPjN3V3ONd+rhoJN2Sv7aVa+4NZbEuWMM1HIfgm22NQ=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM=
 github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 h1:LOd5hxnXDIBcehFrpW1OnXk+VSs0yJXeu1iAOO+Hji4=
 github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417/go.mod h1:btuSv05cYwi9BvLRxVs5zrunU+O1vTgigG1T6UsawcY=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=

internal/storage/bucket/migrations/11-make-stateless/up.sql

@@ -19,7 +19,7 @@ create or replace function transaction_date() returns timestamp as $$
             select ret;
         end if;
 
-        return ret;
+        return ret at time zone 'utc';
     end
 $$ language plpgsql;
 
@@ -33,8 +33,8 @@ alter table moves
 add column transactions_id bigint,
 alter column post_commit_volumes drop not null,
 alter column post_commit_effective_volumes drop not null,
-alter column insertion_date set default (transaction_date() at time zone 'utc'),
-alter column effective_date set default (transaction_date() at time zone 'utc'),
+alter column insertion_date set default transaction_date(),
+alter column effective_date set default transaction_date(),
 alter column account_address_array drop not null;
 
 alter table moves
@@ -60,7 +60,6 @@ from (
       group by move.accounts_address, move.asset
 ) data
 $$ set search_path from current;
-
 create or replace function get_aggregated_effective_volumes_for_transaction(_ledger varchar, tx numeric) returns jsonb
     stable
     language sql
@@ -201,23 +200,25 @@ execute procedure set_compat_on_transactions_metadata();
 
 alter table transactions
 add column post_commit_volumes jsonb,
-add column inserted_at timestamp without time zone default (transaction_date() at time zone 'utc'),
-alter column timestamp set default (transaction_date() at time zone 'utc'),
-alter column id type bigint;
-
-create index transactions_sequences on transactions (id, seq);
+-- todo: set in subsequent migration `default transaction_date()`,
+-- otherwise the function is called for every existing lines
+add column inserted_at timestamp without time zone,
+alter column timestamp set default transaction_date()
+-- todo: we should change the type of this column, but actually it cause a full lock of the table
+-- alter column id type bigint
+;
 
 alter table logs
 add column memento bytea,
 add column idempotency_hash bytea,
 alter column hash drop not null,
-alter column date set default (transaction_date() at time zone 'utc');
+alter column date set default transaction_date();
 
 alter table accounts
 alter column address_array drop not null,
-alter column first_usage set default (transaction_date() at time zone 'utc'),
-alter column insertion_date set default (transaction_date() at time zone 'utc'),
-alter column updated_at set default (transaction_date() at time zone 'utc')
+alter column first_usage set default transaction_date(),
+alter column insertion_date set default transaction_date(),
+alter column updated_at set default transaction_date()
 ;
 
 create table accounts_volumes (
@@ -230,8 +231,6 @@ create table accounts_volumes (
     primary key (ledger, accounts_address, asset)
 );
 
-create index accounts_sequences on accounts (address, seq);
-
 alter table transactions_metadata
 add column transactions_id bigint;
 
@@ -364,6 +363,25 @@ from (select row_number() over () as number, v.value
             select null) v) data
 $$ set search_path from current;
 
+-- todo(next-minor): remove that on future version when the table will have this default value (need to fill nulls before)
+create or replace function set_transaction_inserted_at() returns trigger
+	security definer
+	language plpgsql
+as
+$$
+begin
+	new.inserted_at = transaction_date();
+
+	return new;
+end
+$$ set search_path from current;
+
+create trigger set_transaction_inserted_at
+before insert on transactions
+for each row
+when ( new.inserted_at is null )
+execute procedure set_transaction_inserted_at();
+
 create or replace function set_transaction_addresses() returns trigger
 	security definer
 	language plpgsql
@@ -469,7 +487,6 @@ $do$
 			execute vsql;
 
 			vsql = 'select setval(''"transaction_id_' || ledger.id || '"'', coalesce((select max(id) + 1 from transactions where ledger = ''' || ledger.name || '''), 1)::bigint, false)';
-			raise info '%', vsql;
 			execute vsql;
 
 			-- create a sequence for logs by ledger instead of a sequence of the table as we want to have contiguous ids
@@ -521,9 +538,7 @@ create or replace function enforce_reference_uniqueness() returns trigger
 as
 $$
 begin
-	-- Temporary magic number
-	-- The migration 13 will remove the trigger
-	perform pg_advisory_xact_lock(9999999);
+	perform pg_advisory_xact_lock(hashtext('reference-check' || current_schema));
 
 	if exists(
 		select 1
@@ -544,4 +559,6 @@ after insert on transactions
 deferrable initially deferred
 for each row
 when ( new.reference is not null )
-execute procedure enforce_reference_uniqueness();
+execute procedure enforce_reference_uniqueness();
+
+

internal/storage/bucket/migrations/12-transaction-sequence-index/notes.yaml

@@ -0,0 +1 @@
+name: Create transaction sequences index concurrently

internal/storage/bucket/migrations/12-transaction-sequence-index/up.sql

@@ -0,0 +1 @@
+create index concurrently transactions_sequences on "{{.Schema}}".transactions (id, seq);

internal/storage/bucket/migrations/13-accounts-sequence-index/notes.yaml

@@ -0,0 +1 @@
+name: Create accounts sequences index concurrently

internal/storage/bucket/migrations/13-accounts-sequence-index/up.sql

@@ -0,0 +1 @@
+create index concurrently accounts_sequences on "{{.Schema}}".accounts (address, seq);

internal/storage/ledger/transactions.go

@@ -158,7 +158,7 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti
 							`),
 					).
 					Column("transactions_id").
-					ColumnExpr("aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes").
+					ColumnExpr("public.aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes").
 					Group("transactions_id"),
 			).
 			ColumnExpr("pcev.*")
@@ -241,24 +241,6 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti
 
 func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) error {
 
-	// todo(next-minor): remove that on ledger 2.3 when the corresponding index will be completely built (see migration 12)
-	//if tx.Reference != "" {
-	//	// Magic number, as long as no other process try to take the same exact lock for another reason, it will be ok.
-	//	// This code will be removed in the next minor by the way.
-	//	_, err := s.db.ExecContext(ctx, `select pg_advisory_xact_lock(99999999999)`)
-	//	if err != nil {
-	//		return err
-	//	}
-	//
-	//	exists, err := s.db.NewSelect().
-	//		ModelTableExpr(s.GetPrefixedRelationName("transactions")).
-	//		Where("reference = ?", tx.Reference).
-	//		Exists(ctx)
-	//	if exists {
-	//		return ledgercontroller.NewErrTransactionReferenceConflict(tx.Reference)
-	//	}
-	//}
-
 	postCommitVolumes, err := s.UpdateVolumes(ctx, tx.VolumeUpdates()...)
 	if err != nil {
 		return fmt.Errorf("failed to update balances: %w", err)

internal/storage/ledger/transactions_test.go

@@ -433,7 +433,7 @@ func TestTransactionsCommit(t *testing.T) {
 				errChan <- nil
 			})
 		}
-		wp.StopAndWaitFor(2 * time.Second)
+		wp.StopAndWait()
 		close(errChan)
 
 		for err := range errChan {