diff --git a/example/pgsql8/run b/example/pgsql8/run index 851c185..8b15459 100755 --- a/example/pgsql8/run +++ b/example/pgsql8/run @@ -53,9 +53,9 @@ function pgsql-start { #: Starts and initializes a postgres docker container function pgsql-init { #: Initializes a postgres db info "Initializing db" - psql -d postgres -c 'CREATE ROLE someapp' || continue - psql -d postgres -c 'CREATE ROLE someapp_readonly' || continue - psql -d postgres -c 'CREATE ROLE pgsql' || continue + psql -d postgres -c 'CREATE ROLE someapp' || true + psql -d postgres -c 'CREATE ROLE someapp_readonly' || true + psql -d postgres -c 'CREATE ROLE pgsql' || true } function pgsql-reset { #: Resets the postgres db to a fresh state @@ -90,9 +90,7 @@ function init-db { #: Initializes the database if [[ "$docker" = "enable" ]]; then pgsql-start else - echo "init local DB" pgsql-init - echo "Reset local DB" pgsql-reset fi } diff --git a/example/pgsql8/someapp_extracted_upgrade_stage1_schema1.sql b/example/pgsql8/someapp_extracted_upgrade_stage1_schema1.sql index dba7107..2d40b41 100644 --- a/example/pgsql8/someapp_extracted_upgrade_stage1_schema1.sql +++ b/example/pgsql8/someapp_extracted_upgrade_stage1_schema1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_extracted_upgrade_stage1_schema1.sql --- DBSteward stage 1 structure additions and modifications - generated Thu, 18 Apr 2024 11:08:59 -0400 +-- DBSteward stage 1 structure additions and modifications - generated Thu, 18 Apr 2024 11:43:16 -0400 -- Old definition: pgsql8/someapp_v2_composite.xml -- New definition pgsql8/someapp_extracted_composite.xml diff --git a/example/pgsql8/someapp_extracted_upgrade_stage2_data1.sql b/example/pgsql8/someapp_extracted_upgrade_stage2_data1.sql index 35a0956..0406384 100644 --- a/example/pgsql8/someapp_extracted_upgrade_stage2_data1.sql +++ b/example/pgsql8/someapp_extracted_upgrade_stage2_data1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_extracted_upgrade_stage2_data1.sql --- DBSteward stage 2 data definitions removed - generated Thu, 18 Apr 2024 11:08:59 -0400 +-- DBSteward stage 2 data definitions removed - generated Thu, 18 Apr 2024 11:43:16 -0400 -- Old definition: pgsql8/someapp_v2_composite.xml -- New definition pgsql8/someapp_extracted_composite.xml diff --git a/example/pgsql8/someapp_extracted_upgrade_stage3_schema1.sql b/example/pgsql8/someapp_extracted_upgrade_stage3_schema1.sql index 1769aad..d1ada89 100644 --- a/example/pgsql8/someapp_extracted_upgrade_stage3_schema1.sql +++ b/example/pgsql8/someapp_extracted_upgrade_stage3_schema1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_extracted_upgrade_stage3_schema1.sql --- DBSteward stage 3 structure changes, constraints, and removals - generated Thu, 18 Apr 2024 11:08:59 -0400 +-- DBSteward stage 3 structure changes, constraints, and removals - generated Thu, 18 Apr 2024 11:43:16 -0400 -- Old definition: pgsql8/someapp_v2_composite.xml -- New definition pgsql8/someapp_extracted_composite.xml diff --git a/example/pgsql8/someapp_v1_build.sql b/example/pgsql8/someapp_v1_build.sql index e813803..8a32fad 100644 --- a/example/pgsql8/someapp_v1_build.sql +++ b/example/pgsql8/someapp_v1_build.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_v1_build.sql --- full database definition file generated Thu, 18 Apr 2024 11:08:58 -0400 +-- full database definition file generated Thu, 18 Apr 2024 11:43:15 -0400 BEGIN; @@ -131,3 +131,207 @@ INHERITS (public.sql_user); ALTER TABLE _p_public_sql_user.partition_0 OWNER TO pgsql; +CREATE INDEX user_name_p0 ON _p_public_sql_user.partition_0 USING btree (user_name); + +GRANT SELECT, UPDATE, DELETE, INSERT ON TABLE _p_public_sql_user.partition_0 TO someapp; + +GRANT SELECT ON TABLE _p_public_sql_user.partition_0 TO someapp_readonly; + +CREATE TABLE _p_public_sql_user.partition_1() +INHERITS (public.sql_user); + +ALTER TABLE _p_public_sql_user.partition_1 + OWNER TO pgsql; + +CREATE INDEX user_name_p1 ON _p_public_sql_user.partition_1 USING btree (user_name); + +GRANT SELECT, UPDATE, DELETE, INSERT ON TABLE _p_public_sql_user.partition_1 TO someapp; + +GRANT SELECT ON TABLE _p_public_sql_user.partition_1 TO someapp_readonly; + +CREATE TABLE _p_public_sql_user.partition_2() +INHERITS (public.sql_user); + +ALTER TABLE _p_public_sql_user.partition_2 + OWNER TO pgsql; + +CREATE INDEX user_name_p2 ON _p_public_sql_user.partition_2 USING btree (user_name); + +GRANT SELECT, UPDATE, DELETE, INSERT ON TABLE _p_public_sql_user.partition_2 TO someapp; + +GRANT SELECT ON TABLE _p_public_sql_user.partition_2 TO someapp_readonly; + +CREATE TABLE _p_public_sql_user.partition_3() +INHERITS (public.sql_user); + +ALTER TABLE _p_public_sql_user.partition_3 + OWNER TO pgsql; + +CREATE INDEX user_name_p3 ON _p_public_sql_user.partition_3 USING btree (user_name); + +GRANT SELECT, UPDATE, DELETE, INSERT ON TABLE _p_public_sql_user.partition_3 TO someapp; + +GRANT SELECT ON TABLE _p_public_sql_user.partition_3 TO someapp_readonly; + +CREATE OR REPLACE FUNCTION public.destroy_session(IN character varying) RETURNS VOID +AS $_$ + DELETE FROM session_information WHERE session_id=$1; +$_$ +LANGUAGE sql; + +ALTER FUNCTION public.destroy_session(IN character varying) OWNER TO pgsql; + +COMMENT ON FUNCTION public.destroy_session(IN character varying) IS 'Deletes session data from the database'; + +GRANT EXECUTE ON FUNCTION public.destroy_session(character varying) TO someapp; + +CREATE OR REPLACE FUNCTION _p_public_sql_user.insert_trigger() RETURNS TRIGGER +AS $_$ + DECLARE + mod_result INT; + BEGIN + mod_result := NEW.user_id % 4; + IF (mod_result = 0) THEN + INSERT INTO _p_public_sql_user.partition_0 VALUES (NEW.*); + ELSEIF (mod_result = 1) THEN + INSERT INTO _p_public_sql_user.partition_1 VALUES (NEW.*); + ELSEIF (mod_result = 2) THEN + INSERT INTO _p_public_sql_user.partition_2 VALUES (NEW.*); + ELSEIF (mod_result = 3) THEN + INSERT INTO _p_public_sql_user.partition_3 VALUES (NEW.*); + END IF; + RETURN NULL; + END; +$_$ +LANGUAGE plpgsql; + +ALTER FUNCTION _p_public_sql_user.insert_trigger() OWNER TO pgsql; + +COMMENT ON FUNCTION _p_public_sql_user.insert_trigger() IS 'DBSteward auto-generated for table partition of public.sql_user'; + +GRANT EXECUTE ON FUNCTION _p_public_sql_user.insert_trigger() TO someapp; + +ALTER TABLE public.user_status_list + ALTER COLUMN is_visible SET DEFAULT true; + +ALTER TABLE public.user_status_list + ALTER COLUMN is_visible SET NOT NULL; + +ALTER TABLE public.user_status_list + ALTER COLUMN can_login SET DEFAULT true; + +ALTER TABLE public.user_status_list + ALTER COLUMN can_login SET NOT NULL; + +ALTER TABLE public.user_status_list + ALTER COLUMN user_status SET NOT NULL; + +ALTER TABLE public.session_information + ALTER COLUMN session_id SET NOT NULL; + +ALTER TABLE public.group_list + ALTER COLUMN group_create_time SET NOT NULL; + +ALTER TABLE public.group_list + ALTER COLUMN group_permission SET DEFAULT true; + +ALTER TABLE public.group_list + ALTER COLUMN group_deleted SET DEFAULT false; + +ALTER TABLE public.sql_user + ADD CONSTRAINT sql_user_pkey PRIMARY KEY (user_id); + +ALTER TABLE public.user_status_list + ADD CONSTRAINT user_status_list_pkey PRIMARY KEY (user_status_list_id); + +ALTER TABLE public.session_information + ADD CONSTRAINT session_information_pkey PRIMARY KEY (session_id); + +ALTER TABLE public.group_list + ADD CONSTRAINT group_list_pkey PRIMARY KEY (group_list_id); + +ALTER TABLE _p_public_sql_user.partition_0 + ADD CONSTRAINT partition_0_pkey PRIMARY KEY (user_id); + +ALTER TABLE _p_public_sql_user.partition_1 + ADD CONSTRAINT partition_1_pkey PRIMARY KEY (user_id); + +ALTER TABLE _p_public_sql_user.partition_2 + ADD CONSTRAINT partition_2_pkey PRIMARY KEY (user_id); + +ALTER TABLE _p_public_sql_user.partition_3 + ADD CONSTRAINT partition_3_pkey PRIMARY KEY (user_id); + +ALTER TABLE public.sql_user + ADD CONSTRAINT user_name_unq UNIQUE (user_name); + +ALTER TABLE public.sql_user + ADD CONSTRAINT sql_user_user_status_list_id_fkey FOREIGN KEY (user_status_list_id) REFERENCES public.user_status_list (user_status_list_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +ALTER TABLE _p_public_sql_user.partition_0 + ADD CONSTRAINT sql_user_p_0_chk CHECK ((user_id % 4) = 0); + +ALTER TABLE _p_public_sql_user.partition_0 + ADD CONSTRAINT p0_user_name_unq UNIQUE (user_name); + +ALTER TABLE _p_public_sql_user.partition_0 + ADD CONSTRAINT p0_user_status_list_id_fk FOREIGN KEY (user_status_list_id) REFERENCES public.user_status_list (user_status_list_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +ALTER TABLE _p_public_sql_user.partition_1 + ADD CONSTRAINT sql_user_p_1_chk CHECK ((user_id % 4) = 1); + +ALTER TABLE _p_public_sql_user.partition_1 + ADD CONSTRAINT p1_user_name_unq UNIQUE (user_name); + +ALTER TABLE _p_public_sql_user.partition_1 + ADD CONSTRAINT p1_user_status_list_id_fk FOREIGN KEY (user_status_list_id) REFERENCES public.user_status_list (user_status_list_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +ALTER TABLE _p_public_sql_user.partition_2 + ADD CONSTRAINT sql_user_p_2_chk CHECK ((user_id % 4) = 2); + +ALTER TABLE _p_public_sql_user.partition_2 + ADD CONSTRAINT p2_user_name_unq UNIQUE (user_name); + +ALTER TABLE _p_public_sql_user.partition_2 + ADD CONSTRAINT p2_user_status_list_id_fk FOREIGN KEY (user_status_list_id) REFERENCES public.user_status_list (user_status_list_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +ALTER TABLE _p_public_sql_user.partition_3 + ADD CONSTRAINT sql_user_p_3_chk CHECK ((user_id % 4) = 3); + +ALTER TABLE _p_public_sql_user.partition_3 + ADD CONSTRAINT p3_user_name_unq UNIQUE (user_name); + +ALTER TABLE _p_public_sql_user.partition_3 + ADD CONSTRAINT p3_user_status_list_id_fk FOREIGN KEY (user_status_list_id) REFERENCES public.user_status_list (user_status_list_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +ALTER TABLE public.session_information + ADD CONSTRAINT session_information_user_id_fkey FOREIGN KEY (user_id) REFERENCES public.sql_user (user_id) ON UPDATE NO ACTION ON DELETE NO ACTION; + +CREATE TRIGGER sql_user_part_trg + BEFORE INSERT + ON public.sql_user + FOR EACH ROW + EXECUTE PROCEDURE _p_public_sql_user.insert_trigger(); + +CREATE OR REPLACE VIEW public.group_list_view AS + SELECT * FROM public.group_list WHERE group_deleted = FALSE; + +ALTER VIEW public.group_list_view OWNER TO pgsql; + +GRANT SELECT ON TABLE public.group_list_view TO someapp; + +INSERT INTO public.user_status_list (user_status_list_id, user_status, is_visible, can_login) VALUES (1, 'Active', 'true', 'true'); + +INSERT INTO public.user_status_list (user_status_list_id, user_status, is_visible, can_login) VALUES (2, 'Inactive', 'false', 'true'); + +INSERT INTO public.user_status_list (user_status_list_id, user_status, is_visible, can_login) VALUES (3, 'Closed', 'false', 'false'); + +INSERT INTO public.sql_user (user_id, user_name, password, user_status_list_id, import_id, register_date) VALUES (1, 'someapp_admin', '7c6a180b36896a0a8c02787eeafb0e4c', 3, 'DEFAULT_USER', (NOW())); + +SELECT setval(pg_get_serial_sequence('public.sql_user', 'user_id'), MAX(user_id), true) FROM public.sql_user; + + +-- NON-STAGED SQL COMMANDS + +COMMIT; + diff --git a/example/pgsql8/someapp_v2_upgrade_stage1_schema1.sql b/example/pgsql8/someapp_v2_upgrade_stage1_schema1.sql index f03c502..e677b12 100644 --- a/example/pgsql8/someapp_v2_upgrade_stage1_schema1.sql +++ b/example/pgsql8/someapp_v2_upgrade_stage1_schema1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_v2_upgrade_stage1_schema1.sql --- DBSteward stage 1 structure additions and modifications - generated Thu, 18 Apr 2024 11:08:58 -0400 +-- DBSteward stage 1 structure additions and modifications - generated Thu, 18 Apr 2024 11:43:15 -0400 -- Old definition: pgsql8/someapp_v1_composite.xml -- New definition pgsql8/someapp_v2_composite.xml diff --git a/example/pgsql8/someapp_v2_upgrade_stage2_data1.sql b/example/pgsql8/someapp_v2_upgrade_stage2_data1.sql index 7314fe5..b709473 100644 --- a/example/pgsql8/someapp_v2_upgrade_stage2_data1.sql +++ b/example/pgsql8/someapp_v2_upgrade_stage2_data1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_v2_upgrade_stage2_data1.sql --- DBSteward stage 2 data definitions removed - generated Thu, 18 Apr 2024 11:08:58 -0400 +-- DBSteward stage 2 data definitions removed - generated Thu, 18 Apr 2024 11:43:15 -0400 -- Old definition: pgsql8/someapp_v1_composite.xml -- New definition pgsql8/someapp_v2_composite.xml diff --git a/example/pgsql8/someapp_v2_upgrade_stage3_schema1.sql b/example/pgsql8/someapp_v2_upgrade_stage3_schema1.sql index e04511b..5373e9d 100644 --- a/example/pgsql8/someapp_v2_upgrade_stage3_schema1.sql +++ b/example/pgsql8/someapp_v2_upgrade_stage3_schema1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_v2_upgrade_stage3_schema1.sql --- DBSteward stage 3 structure changes, constraints, and removals - generated Thu, 18 Apr 2024 11:08:58 -0400 +-- DBSteward stage 3 structure changes, constraints, and removals - generated Thu, 18 Apr 2024 11:43:15 -0400 -- Old definition: pgsql8/someapp_v1_composite.xml -- New definition pgsql8/someapp_v2_composite.xml diff --git a/example/pgsql8/someapp_v2_upgrade_stage4_data1.sql b/example/pgsql8/someapp_v2_upgrade_stage4_data1.sql index 4b992a4..2a3bc78 100644 --- a/example/pgsql8/someapp_v2_upgrade_stage4_data1.sql +++ b/example/pgsql8/someapp_v2_upgrade_stage4_data1.sql @@ -1,5 +1,5 @@ -- pgsql8/someapp_v2_upgrade_stage4_data1.sql --- DBSteward stage 4 data definition changes and additions - generated Thu, 18 Apr 2024 11:08:58 -0400 +-- DBSteward stage 4 data definition changes and additions - generated Thu, 18 Apr 2024 11:43:15 -0400 -- Old definition: pgsql8/someapp_v1_composite.xml -- New definition pgsql8/someapp_v2_composite.xml diff --git a/lib/encoding/xml/index.go b/lib/encoding/xml/index.go index 2597909..b5ddaf1 100644 --- a/lib/encoding/xml/index.go +++ b/lib/encoding/xml/index.go @@ -23,11 +23,31 @@ type IndexDim struct { Value string `xml:",chardata"` } +func (id *IndexDim) ToModel() (*model.IndexDim, error) { + return &model.IndexDim{ + Name: id.Name, + Sql: id.Sql, + Value: id.Value, + }, nil +} + type IndexCond struct { SqlFormat string `xml:"sqlFormat,attr,omitempty"` Condition string `xml:",chardata"` } +func (id *IndexCond) ToModel() (*model.IndexCond, error) { + rv := model.IndexCond{ + Condition: id.Condition, + } + var err error + rv.SqlFormat, err = model.NewSqlFormat(id.SqlFormat) + if err != nil { + return nil, err + } + return &rv, nil +} + func (idx *Index) ToModel() (*model.Index, error) { rv := model.Index{ Name: idx.Name, @@ -39,6 +59,20 @@ func (idx *Index) ToModel() (*model.Index, error) { if err != nil { return nil, fmt.Errorf("index '%s' invalid: %s", idx.Name, err) } + for _, d := range idx.Dimensions { + nd, err := d.ToModel() + if err != nil { + return nil, fmt.Errorf("index '%s' invalid: %s", idx.Name, err) + } + rv.Dimensions = append(rv.Dimensions, nd) + } + for _, c := range idx.Conditions { + nc, err := c.ToModel() + if err != nil { + return nil, fmt.Errorf("index '%s' invalid: %s", idx.Name, err) + } + rv.Conditions = append(rv.Conditions, nc) + } return &rv, nil } diff --git a/lib/format/pgsql8/live/types.go b/lib/format/pgsql8/live/types.go index 47ad691..2ce16c2 100644 --- a/lib/format/pgsql8/live/types.go +++ b/lib/format/pgsql8/live/types.go @@ -1,7 +1,8 @@ package live import ( - "github.com/dbsteward/dbsteward/lib/util" + "database/sql" + "github.com/jackc/pgtype" ) @@ -36,11 +37,11 @@ type SequenceRelEntry struct { } type SequenceEntry struct { - Cache util.Opt[int] - Start util.Opt[int] - Min util.Opt[int] - Max util.Opt[int] - Increment util.Opt[int] + Cache sql.NullInt64 + Start sql.NullInt64 + Min sql.NullInt64 + Max sql.NullInt64 + Increment sql.NullInt64 Cycled bool } diff --git a/lib/format/pgsql8/operations.go b/lib/format/pgsql8/operations.go index 6b252b0..dc66b37 100644 --- a/lib/format/pgsql8/operations.go +++ b/lib/format/pgsql8/operations.go @@ -379,11 +379,11 @@ func (self *Operations) ExtractSchema(host string, port uint, name, user, pass s schema.AddSequence(&model.Sequence{ Name: seqListRow.Name, Owner: seqListRow.Owner, // TODO(feat) should this have a translateRoleName call? - Cache: seqRow.Cache, - Start: seqRow.Start, - Min: seqRow.Min, - Max: seqRow.Max, - Increment: seqRow.Increment, + Cache: util.OptFromSQLNullInt64(seqRow.Cache), + Start: util.OptFromSQLNullInt64(seqRow.Start), + Min: util.OptFromSQLNullInt64(seqRow.Min), + Max: util.OptFromSQLNullInt64(seqRow.Max), + Increment: util.OptFromSQLNullInt64(seqRow.Increment), Cycle: seqRow.Cycled, }) } diff --git a/lib/format/pgsql8/operations_extract_schema_test.go b/lib/format/pgsql8/operations_extract_schema_test.go index eda63e9..69f43e4 100644 --- a/lib/format/pgsql8/operations_extract_schema_test.go +++ b/lib/format/pgsql8/operations_extract_schema_test.go @@ -1,6 +1,7 @@ package pgsql8_test import ( + "database/sql" "strings" "testing" @@ -549,10 +550,10 @@ func TestOperations_ExtractSchema_Sequences(t *testing.T) { {Name: "blah", Owner: "owner"}, }, nil) introspector.EXPECT().GetSequencesForRel("public", "test_seq").Return([]live.SequenceEntry{ - {Start: util.Some(1), Increment: util.Some(1), Cache: util.Some(1), Max: util.Some(15)}, + {Start: sql.NullInt64{1, true}, Increment: sql.NullInt64{1, true}, Cache: sql.NullInt64{1, true}, Max: sql.NullInt64{15, true}}, }, nil) introspector.EXPECT().GetSequencesForRel("public", "blah").Return([]live.SequenceEntry{ - {Cache: util.Some(5), Min: util.Some(3), Max: util.Some(10)}, + {Cache: sql.NullInt64{5, true}, Min: sql.NullInt64{3, true}, Max: sql.NullInt64{10, true}}, }, nil) introspector.EXPECT().GetIndexes(gomock.Any(), gomock.Any()).AnyTimes() introspector.EXPECT().GetConstraints().Return([]live.ConstraintEntry{ diff --git a/lib/format/pgsql8/xml_parser_partition_modulo.go b/lib/format/pgsql8/xml_parser_partition_modulo.go index 27ae343..dafdbf5 100644 --- a/lib/format/pgsql8/xml_parser_partition_modulo.go +++ b/lib/format/pgsql8/xml_parser_partition_modulo.go @@ -2,6 +2,7 @@ package pgsql8 import ( "fmt" + "log" "strconv" "github.com/dbsteward/dbsteward/lib" @@ -90,6 +91,9 @@ func (self *XmlParser) createModuloPartitionTables(schema *model.Schema, table * }) for _, index := range table.Indexes { + if len(index.Dimensions) == 0 { + log.Panicf("Index %s has no dimensions", index.Name) + } indexCopy := *index indexCopy.Name = simpleBuildIdentifier("", index.Name, "_p"+partNum) partTable.AddIndex(&indexCopy) diff --git a/lib/util/optional.go b/lib/util/optional.go index c062b0e..c42c4eb 100644 --- a/lib/util/optional.go +++ b/lib/util/optional.go @@ -1,12 +1,22 @@ package util -import "fmt" +import ( + "database/sql" + "fmt" +) type Opt[T any] struct { hasValue bool value T } +func OptFromSQLNullInt64(t sql.NullInt64) Opt[int] { + if t.Valid { + return Opt[int]{true, int(t.Int64)} + } + return Opt[int]{false, 0} +} + func Some[T any](t T) Opt[T] { return Opt[T]{true, t} }