From 5314238836a323691f5956a11b19df86bf532182 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 30 Oct 2024 18:32:27 +0100 Subject: [PATCH] feat: move long migrations in dedicated dir --- docker-compose.yml | 2 - go.mod | 2 +- go.sum | 4 ++ .../controller/ledger/store_generated_test.go | 4 +- .../migrations/11-make-stateless/up.sql | 57 ++++++++++++------- .../12-transaction-sequence-index/notes.yaml | 1 + .../12-transaction-sequence-index/up.sql | 1 + .../13-accounts-sequence-index/notes.yaml | 1 + .../13-accounts-sequence-index/up.sql | 1 + .../notes.yaml | 0 .../up.sql | 0 .../notes.yaml | 0 .../up.sql | 0 internal/storage/ledger/transactions.go | 20 +------ internal/storage/ledger/transactions_test.go | 2 +- 15 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 internal/storage/bucket/migrations/12-transaction-sequence-index/notes.yaml create mode 100644 internal/storage/bucket/migrations/12-transaction-sequence-index/up.sql create mode 100644 internal/storage/bucket/migrations/13-accounts-sequence-index/notes.yaml create mode 100644 internal/storage/bucket/migrations/13-accounts-sequence-index/up.sql rename internal/storage/bucket/migrations/{12-transaction-reference-index => 14-transaction-reference-index}/notes.yaml (100%) rename internal/storage/bucket/migrations/{12-transaction-reference-index => 14-transaction-reference-index}/up.sql (100%) rename internal/storage/bucket/migrations/{13-create-ledger-indexes => 15-create-ledger-indexes}/notes.yaml (100%) rename internal/storage/bucket/migrations/{13-create-ledger-indexes => 15-create-ledger-indexes}/up.sql (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 85ba8ec86..d315ca717 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - volumes: postgres: {} diff --git a/go.mod b/go.mod index b7dc25055..57cdb46a0 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 github.com/bluele/gcache v0.0.2 github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3 - github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7 + github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289 github.com/formancehq/ledger/pkg/client v0.0.0-00010101000000-000000000000 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/cors v1.2.1 diff --git a/go.sum b/go.sum index e61022ae2..7370be6e3 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,10 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7 h1:OZz4N9nIj814aIgpqIvojndtae+N9Vqj5MJgKPIzJ5Y= github.com/formancehq/go-libs/v2 v2.0.1-0.20241029111513-edb146ee0db7/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241030160027-898dbd1a42af h1:7POEnA2uHO+a8HNO+LGCuSVpBolflAFcNR0N1BhEuRA= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241030160027-898dbd1a42af/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289 h1:XPjN3V3ONd+rhoJN2Sv7aVa+4NZbEuWMM1HIfgm22NQ= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241030165600-9060fd311289/go.mod h1:DTqSp28pYPZa4O1WrOg3kobhgTHdk9geGtxnws9EViM= github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 h1:LOd5hxnXDIBcehFrpW1OnXk+VSs0yJXeu1iAOO+Hji4= github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417/go.mod h1:btuSv05cYwi9BvLRxVs5zrunU+O1vTgigG1T6UsawcY= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= diff --git a/internal/controller/ledger/store_generated_test.go b/internal/controller/ledger/store_generated_test.go index 59479b0da..1618ae7c3 100644 --- a/internal/controller/ledger/store_generated_test.go +++ b/internal/controller/ledger/store_generated_test.go @@ -364,7 +364,7 @@ func (mr *MockStoreMockRecorder) GetVolumesWithBalances(ctx, q any) *gomock.Call // IsUpToDate mocks base method. func (m *MockStore) IsUpToDate(ctx context.Context) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HasMinimalVersion", ctx) + ret := m.ctrl.Call(m, "IsUpToDate", ctx) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 @@ -373,7 +373,7 @@ func (m *MockStore) IsUpToDate(ctx context.Context) (bool, error) { // IsUpToDate indicates an expected call of IsUpToDate. func (mr *MockStoreMockRecorder) IsUpToDate(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasMinimalVersion", reflect.TypeOf((*MockStore)(nil).IsUpToDate), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUpToDate", reflect.TypeOf((*MockStore)(nil).IsUpToDate), ctx) } // ListAccounts mocks base method. diff --git a/internal/storage/bucket/migrations/11-make-stateless/up.sql b/internal/storage/bucket/migrations/11-make-stateless/up.sql index b8d2c9666..182ce53b4 100644 --- a/internal/storage/bucket/migrations/11-make-stateless/up.sql +++ b/internal/storage/bucket/migrations/11-make-stateless/up.sql @@ -19,7 +19,7 @@ create or replace function transaction_date() returns timestamp as $$ select ret; end if; - return ret; + return ret at time zone 'utc'; end $$ language plpgsql; @@ -33,8 +33,8 @@ alter table moves add column transactions_id bigint, alter column post_commit_volumes drop not null, alter column post_commit_effective_volumes drop not null, -alter column insertion_date set default (transaction_date() at time zone 'utc'), -alter column effective_date set default (transaction_date() at time zone 'utc'), +alter column insertion_date set default transaction_date(), +alter column effective_date set default transaction_date(), alter column account_address_array drop not null; alter table moves @@ -60,7 +60,6 @@ from ( group by move.accounts_address, move.asset ) data $$ set search_path from current; - create or replace function get_aggregated_effective_volumes_for_transaction(_ledger varchar, tx numeric) returns jsonb stable language sql @@ -201,23 +200,25 @@ execute procedure set_compat_on_transactions_metadata(); alter table transactions add column post_commit_volumes jsonb, -add column inserted_at timestamp without time zone default (transaction_date() at time zone 'utc'), -alter column timestamp set default (transaction_date() at time zone 'utc'), -alter column id type bigint; - -create index transactions_sequences on transactions (id, seq); +-- todo: set in subsequent migration `default transaction_date()`, +-- otherwise the function is called for every existing lines +add column inserted_at timestamp without time zone, +alter column timestamp set default transaction_date() +-- todo: we should change the type of this column, but actually it cause a full lock of the table +-- alter column id type bigint +; alter table logs add column memento bytea, add column idempotency_hash bytea, alter column hash drop not null, -alter column date set default (transaction_date() at time zone 'utc'); +alter column date set default transaction_date(); alter table accounts alter column address_array drop not null, -alter column first_usage set default (transaction_date() at time zone 'utc'), -alter column insertion_date set default (transaction_date() at time zone 'utc'), -alter column updated_at set default (transaction_date() at time zone 'utc') +alter column first_usage set default transaction_date(), +alter column insertion_date set default transaction_date(), +alter column updated_at set default transaction_date() ; create table accounts_volumes ( @@ -230,8 +231,6 @@ create table accounts_volumes ( primary key (ledger, accounts_address, asset) ); -create index accounts_sequences on accounts (address, seq); - alter table transactions_metadata add column transactions_id bigint; @@ -364,6 +363,25 @@ from (select row_number() over () as number, v.value select null) v) data $$ set search_path from current; +-- todo(next-minor): remove that on future version when the table will have this default value (need to fill nulls before) +create or replace function set_transaction_inserted_at() returns trigger + security definer + language plpgsql +as +$$ +begin + new.inserted_at = transaction_date(); + + return new; +end +$$ set search_path from current; + +create trigger set_transaction_inserted_at +before insert on transactions +for each row +when ( new.inserted_at is null ) +execute procedure set_transaction_inserted_at(); + create or replace function set_transaction_addresses() returns trigger security definer language plpgsql @@ -469,7 +487,6 @@ $do$ execute vsql; vsql = 'select setval(''"transaction_id_' || ledger.id || '"'', coalesce((select max(id) + 1 from transactions where ledger = ''' || ledger.name || '''), 1)::bigint, false)'; - raise info '%', vsql; execute vsql; -- create a sequence for logs by ledger instead of a sequence of the table as we want to have contiguous ids @@ -521,9 +538,7 @@ create or replace function enforce_reference_uniqueness() returns trigger as $$ begin - -- Temporary magic number - -- The migration 13 will remove the trigger - perform pg_advisory_xact_lock(9999999); + perform pg_advisory_xact_lock(hashtext('reference-check' || current_schema)); if exists( select 1 @@ -544,4 +559,6 @@ after insert on transactions deferrable initially deferred for each row when ( new.reference is not null ) -execute procedure enforce_reference_uniqueness(); \ No newline at end of file +execute procedure enforce_reference_uniqueness(); + + diff --git a/internal/storage/bucket/migrations/12-transaction-sequence-index/notes.yaml b/internal/storage/bucket/migrations/12-transaction-sequence-index/notes.yaml new file mode 100644 index 000000000..e1e842c81 --- /dev/null +++ b/internal/storage/bucket/migrations/12-transaction-sequence-index/notes.yaml @@ -0,0 +1 @@ +name: Create transaction sequences index concurrently diff --git a/internal/storage/bucket/migrations/12-transaction-sequence-index/up.sql b/internal/storage/bucket/migrations/12-transaction-sequence-index/up.sql new file mode 100644 index 000000000..53ecd3c68 --- /dev/null +++ b/internal/storage/bucket/migrations/12-transaction-sequence-index/up.sql @@ -0,0 +1 @@ +create index concurrently transactions_sequences on "{{.Schema}}".transactions (id, seq); \ No newline at end of file diff --git a/internal/storage/bucket/migrations/13-accounts-sequence-index/notes.yaml b/internal/storage/bucket/migrations/13-accounts-sequence-index/notes.yaml new file mode 100644 index 000000000..3b8dfcad1 --- /dev/null +++ b/internal/storage/bucket/migrations/13-accounts-sequence-index/notes.yaml @@ -0,0 +1 @@ +name: Create accounts sequences index concurrently diff --git a/internal/storage/bucket/migrations/13-accounts-sequence-index/up.sql b/internal/storage/bucket/migrations/13-accounts-sequence-index/up.sql new file mode 100644 index 000000000..b49646204 --- /dev/null +++ b/internal/storage/bucket/migrations/13-accounts-sequence-index/up.sql @@ -0,0 +1 @@ +create index concurrently accounts_sequences on "{{.Schema}}".accounts (address, seq); \ No newline at end of file diff --git a/internal/storage/bucket/migrations/12-transaction-reference-index/notes.yaml b/internal/storage/bucket/migrations/14-transaction-reference-index/notes.yaml similarity index 100% rename from internal/storage/bucket/migrations/12-transaction-reference-index/notes.yaml rename to internal/storage/bucket/migrations/14-transaction-reference-index/notes.yaml diff --git a/internal/storage/bucket/migrations/12-transaction-reference-index/up.sql b/internal/storage/bucket/migrations/14-transaction-reference-index/up.sql similarity index 100% rename from internal/storage/bucket/migrations/12-transaction-reference-index/up.sql rename to internal/storage/bucket/migrations/14-transaction-reference-index/up.sql diff --git a/internal/storage/bucket/migrations/13-create-ledger-indexes/notes.yaml b/internal/storage/bucket/migrations/15-create-ledger-indexes/notes.yaml similarity index 100% rename from internal/storage/bucket/migrations/13-create-ledger-indexes/notes.yaml rename to internal/storage/bucket/migrations/15-create-ledger-indexes/notes.yaml diff --git a/internal/storage/bucket/migrations/13-create-ledger-indexes/up.sql b/internal/storage/bucket/migrations/15-create-ledger-indexes/up.sql similarity index 100% rename from internal/storage/bucket/migrations/13-create-ledger-indexes/up.sql rename to internal/storage/bucket/migrations/15-create-ledger-indexes/up.sql diff --git a/internal/storage/ledger/transactions.go b/internal/storage/ledger/transactions.go index 40c93c5fb..a8b522e7d 100644 --- a/internal/storage/ledger/transactions.go +++ b/internal/storage/ledger/transactions.go @@ -158,7 +158,7 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti `), ). Column("transactions_id"). - ColumnExpr("aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). + ColumnExpr("public.aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). Group("transactions_id"), ). ColumnExpr("pcev.*") @@ -241,24 +241,6 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) error { - // todo(next-minor): remove that on ledger 2.3 when the corresponding index will be completely built (see migration 12) - //if tx.Reference != "" { - // // Magic number, as long as no other process try to take the same exact lock for another reason, it will be ok. - // // This code will be removed in the next minor by the way. - // _, err := s.db.ExecContext(ctx, `select pg_advisory_xact_lock(99999999999)`) - // if err != nil { - // return err - // } - // - // exists, err := s.db.NewSelect(). - // ModelTableExpr(s.GetPrefixedRelationName("transactions")). - // Where("reference = ?", tx.Reference). - // Exists(ctx) - // if exists { - // return ledgercontroller.NewErrTransactionReferenceConflict(tx.Reference) - // } - //} - postCommitVolumes, err := s.UpdateVolumes(ctx, tx.VolumeUpdates()...) if err != nil { return fmt.Errorf("failed to update balances: %w", err) diff --git a/internal/storage/ledger/transactions_test.go b/internal/storage/ledger/transactions_test.go index 5978f51a0..6bb0312e3 100644 --- a/internal/storage/ledger/transactions_test.go +++ b/internal/storage/ledger/transactions_test.go @@ -433,7 +433,7 @@ func TestTransactionsCommit(t *testing.T) { errChan <- nil }) } - wp.StopAndWaitFor(2 * time.Second) + wp.StopAndWait() close(errChan) for err := range errChan {