scripts to fix db

.env.local-dev

@@ -1,7 +1,16 @@
 ENV=LOCAL
 
+#DB_USER=postgres
+#DB_PASSWORD=password
+#DB_HOST=localhost
+#DB_PORT=5234
+#DB_NAME=dusa_dev_db
+
 DB_USER=postgres
-DB_PASSWORD=password
-DB_HOST=localhost
-DB_PORT=5234
-DB_NAME=dusa_dev_db
+DB_PASSWORD=H6k+vzk_aVVxi9\a
+DB_HOST=10.4.208.3
+DB_PORT=3306
+DB_NAME=dusa_db
+GOOGLE_CLOUD_PROJECT=down-under-stats-app
+GOOGLE_CLOUD_REGION=europe-west1
+CLOUD_SQL_INSTANCE=dusa-backend-postgres

docker/docker-compose.dev.yaml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   dusa_dev_db:
-    image: postgres:latest
+    image: postgres:15.6
     restart: always
     ports:
       - "5234:5432"

manage.py

@@ -1,6 +1,15 @@
 from typer import Typer
 
 from src.dusa_backend.scripts.populate_db import populate_db as _populate_database
+from src.dusa_backend.scripts.remove_duplicate_stats import (
+    remove_duplicate_health_stats as _remove_duplicate_health_stats,
+)
+from src.dusa_backend.scripts.remove_records import remove_records as _remove_records
+from src.dusa_backend.scripts.atm_records_fix import atm_records_fix as _atm_records_fix
+from src.dusa_backend.scripts.drinks_created_fix import (
+    drinks_fix_matcher as _drinks_fix_matcher,
+    change_drinks_created as _change_drinks_created,
+)
 
 app = Typer()
 
@@ -11,8 +20,28 @@ def populate_database():
 
 
 @app.command()
-def hello_world():
-    print("Hello World!")
+def remove_duplicate_health_stats():
+    _remove_duplicate_health_stats()
+
+
+@app.command()
+def remove_records():
+    _remove_records()
+
+
+@app.command()
+def atm_records_fix():
+    _atm_records_fix()
+
+
+@app.command()
+def drinks_fix_matcher():
+    _drinks_fix_matcher()
+
+
+@app.command()
+def change_drinks_created():
+    _change_drinks_created()
 
 
 if __name__ == "__main__":

src/dusa_backend/core/settings.py

@@ -18,6 +18,7 @@ class EnvEnum(StrEnum):
 ENV_FILE_MAPPING = {
     EnvEnum.TESTING: BACKEND_DIR / ".env.testing",
     EnvEnum.LOCAL: BACKEND_DIR / ".env.local-dev",
+    EnvEnum.DEV: BACKEND_DIR / ".env.local-dev",
 }
 
 

src/dusa_backend/infrastructure/database/session.py

@@ -38,7 +38,6 @@ def get_cloud_sql_connection() -> Connection:
             user=settings.db_user,
             password=settings.db_password,
             db=settings.db_name,
-            # enable_iam_auth=True,
         )
 
     return create_engine("postgresql+pg8000://", creator=get_cloud_sql_connection)

src/dusa_backend/scripts/atm_records_fix.py

@@ -0,0 +1,23 @@
+from dusa_backend.domain.category_items.repository import CategoryItemRepository
+from dusa_backend.domain.records.repository import RecordRepository
+from dusa_backend.infrastructure.database.session import SessionLocal
+from dusa_backend.infrastructure.database.tables import RecordTable
+
+
+ATM_RECORD_IDS = [
+    "b1444629-e086-40cf-a012-c97760a4f932",
+    "74c49e90-b688-42ee-8c99-8b5fed101ccb",
+    "b00e04b6-45ee-43bd-8db0-4b1fb40b4ea7",
+]
+
+
+def atm_records_fix():
+    with SessionLocal() as db_session:
+        record_repo = RecordRepository(db_session)
+        category_item_repo = CategoryItemRepository(db_session)
+
+        atm_category_item = category_item_repo.get(name="ATM")
+        records = db_session.query(RecordTable).filter(RecordTable.id.in_(ATM_RECORD_IDS))
+        for record in records:
+            record.category_item_id = atm_category_item.id
+            record_repo.update(record)

src/dusa_backend/scripts/drinks_created_fix.py

@@ -0,0 +1,184 @@
+from datetime import datetime, timedelta
+
+import pytz
+
+from dusa_backend.domain.categories.repository import CategoryRepository
+from dusa_backend.domain.category_items.repository import CategoryItemRepository
+from dusa_backend.domain.records.repository import RecordRepository
+from dusa_backend.infrastructure.database.session import SessionLocal
+from dusa_backend.infrastructure.database.tables import RecordTable
+
+
+NSW_TZ = "Australia/Sydney"
+QLD_TZ = "Australia/Brisbane"
+NZ_TZ = "Pacific/Auckland"
+
+CORRECT_RECORD_IDS = [
+    "ae214406-1be6-40a1-9ae5-04e0d1719194",
+    "1b2e3037-c0a5-4b27-afba-8a58e6362bf2",
+    "923c42fa-f133-4c87-a42e-b5df5492c0b5",
+    "af43aa3d-ff91-454d-9835-a82bcef2f150",
+    "aaf751f4-008f-4532-ba6f-05a747beb9a6",
+    "eb025261-0d75-499e-ae4d-d5c9c078dac6",
+    "f0d8877c-1dd8-444e-b4c7-f3088b082fd4",
+    "c450f2e4-6091-4f28-8a9f-f62f4bedce3c",
+    "162dcbae-2368-4cbe-b18e-198e9fb1d004",
+    "14f4962b-803d-4ab9-bd21-880fcd5466d6",
+    "c93c3b09-1fca-4777-8cd6-7e13016af9c2",
+    "7d64370a-726f-442c-9409-63fc8c127492",
+    "6f59e55d-fb26-4076-b90e-455d98e76597",
+    "c18d39cc-9f1e-4b20-831a-ffd52c09e01d",
+    "1d2b21a5-b4f8-40b3-9498-8defecd75f7c",
+    "3c1b4806-8a63-40a4-8439-825e47bbd488",
+    "5cc4ac41-df02-457b-9a13-1967bf28b94a",
+    "8f1ee4a9-37d4-4924-b0fd-d87435b35641",
+    "abfeb145-3374-430a-8c26-1ec030e006f6",
+    "39506353-7018-439c-9c41-491651ef7190",
+    "c285e165-91dc-44fe-ba04-c6766030e8d0",
+    "9f959ea7-8131-4e2d-a348-79a58f3cc692",
+    "ef44bbed-1904-40ed-b782-e01dd2faf3c4",
+    "a716e146-dc2b-4a99-9f22-107eff7c5252",
+    "ae4a538b-9c3a-48a1-9d29-ad166ef16614",
+    "5b881ae3-afd1-43bc-82c6-557bad0ecb11",
+    "38ab542f-3445-412e-87b1-08907f1e3f0d",
+    "e7758a87-e687-4e26-b1e4-b47ec2534fc5",
+    "4d88cf43-a164-4655-a3bb-de17c977db6f",
+    "088e2b12-a2b1-4388-ba2c-18e8c6067a10",
+    "36117810-fe39-45c5-9134-0f37b0586672",
+    "a355ee93-901f-4bda-bcaf-d572fbd7b77c",
+    "0944fbe0-5b2c-448e-a8aa-73cadb103140",
+    "74eaa84b-f172-41cc-af53-75450eb7b609",
+    "94f578ef-9525-45a0-8dec-c767a58d4dc9",
+    "a6028a50-f739-451f-a684-d87daefd9931",
+    "bc9a32b2-9ac8-43c3-a5d2-c8f285c50250",
+    "ae8d9f60-b5ab-439d-a114-9b5d58ede669",
+    "b56bac03-4427-45bc-8a6b-d94c7c9f91ea",
+    "e3ba19cf-e662-477f-8394-c8affeeecec4",
+    "b2e89639-f9a3-4ce0-963b-fc4946c835e5",
+    "76660282-a82b-4d7c-987a-83ae881702f8",
+    "0aac59db-55e2-4d21-b36f-6316a254e7f5",
+    "b6977878-04de-4936-95a2-bf577969e361",
+    "e29bd6fc-d4f3-4a99-92ce-7bca33567a51",
+    "6e47cd9b-50e3-4058-9e0e-8917912cfb13",
+    "6530f25e-ac3e-49a0-895d-0f1c516e9c16",
+    "ccbaac28-64dc-42be-be48-409f8e5a0a75",
+    "c0db5942-e327-46ac-a1b2-7a8a74c79907",
+    "fc2657c2-f092-4b3f-9c5e-bf3af7b5b037",
+    "e79cb9c7-43c5-4d16-98e9-cd958175d7ce",
+    "cfaa6d79-9a86-4230-a061-cef9dfc60d20",
+    "371bbe32-4110-409a-b93d-e908c2b53109",
+    "265303c4-fe18-4afc-bdf4-01106632e070",
+    "97d3f3cf-2b50-4ec5-94cb-a70052bb179f",
+    "7bc429d1-15b6-4f28-b430-4b069fa0281b",
+    "bf9d43a1-34f9-45b2-a59b-86a3198cdd95",
+    "77623fd1-7426-4ac0-862a-cf3e4ed3717a",
+    "e61105d2-3301-4cee-9c43-0b5161e6549d",
+    "a450df16-03b1-4e16-9a59-5f21451cdac5",
+    "44b29482-fee9-421f-8f4d-648b20db9eb7",
+    "ccfb2788-ef88-45e7-a79a-8c8e8c51c132",
+    "caf79faa-d7f9-4fa4-9a54-9f4b81481c95",
+    "32c8e319-254c-41d8-b743-380f7d130722",
+    "6ccbf6a2-7bdb-47cf-91a1-84d954542e3f",
+    "7d5712c3-ee44-4667-88e4-8fea65f1721c",
+    "a98ef6cd-99e1-43da-a0b1-b7c995d1cba5",
+    "676f2d32-37c1-4f16-85f3-4bd30bf53e2f",
+    "d2e733a5-35d7-4ebc-970d-d5010fa8a171",
+    "c2ba3ed1-b48b-465c-8a09-28db535a35fc",
+    "bed3c38a-2364-4ab7-9570-4b372220be8a",
+    "06e2da33-59d9-46ad-ad13-5f5a50748d1d",
+    "8cedbd73-e73d-42ff-9911-fd78423fc394",
+    "bef95a70-8bb7-4491-a1d1-5c594ce5b7af",
+    "dad30166-8cea-4e74-a21e-356e1eb828a9",
+    "872c4190-7d04-4fb9-a1d4-26fe7d86373a",
+    "7cdebdb5-b70a-486e-9b4b-78555371408f",
+    "ed90db53-f93c-44e6-ab96-9f39d4d85dd3",
+    "282771b9-b660-435f-848e-379dc831956b",
+    "35e60056-6377-4521-8978-5ae96c738514",
+    "6e10c2e2-eecf-465b-b06c-0f010eddd3c2",
+    "3a708f66-dc2a-4d74-96f3-db23d5e742ce",
+]
+
+
+def get_created_tz(created):
+    tz = NSW_TZ
+    if datetime(2024, 2, 4) <= created < datetime(2024, 2, 24):
+        tz = QLD_TZ
+    elif datetime(2024, 2, 24) <= created < datetime(2024, 2, 26):
+        tz = NSW_TZ
+    elif datetime(2024, 2, 26) <= created < datetime(2024, 3, 17):
+        tz = NZ_TZ
+    return created.astimezone(pytz.timezone(tz))
+
+
+def drinks_fix_matcher():
+    correct_drink_ids = []
+
+    def print_days_purchases(records, day):
+        for record in records:
+            if record.created.date() == day:
+                print(record.category_item.name, get_created_tz(record.created), record.value)
+
+    with SessionLocal() as db_session:
+        category_repo = CategoryRepository(db_session)
+        category_item_repo = CategoryItemRepository(db_session)
+
+        drinks_category = category_repo.get(name="Drinks")
+        drinks_category_item = category_item_repo.get(name="Drinks/Nights Out")
+        food_category_item = category_item_repo.get(name="Food and/or drink")
+        money_spent_records = (
+            db_session.query(RecordTable)
+            .join(RecordTable.category_item)
+            .filter(RecordTable.category_item_id.in_([drinks_category_item.id, food_category_item.id]))
+            .order_by(RecordTable.created.asc())
+        )
+        drinks_records = (
+            db_session.query(RecordTable)
+            .join(RecordTable.category_item)
+            .filter(RecordTable.category_item_id.in_([dc_item.id for dc_item in drinks_category.category_items]))
+            .order_by(RecordTable.created.asc())
+        )
+
+        created_drinks_mapper = {}
+        for dr in drinks_records:
+            if dr.created.date() in created_drinks_mapper:
+                created_drinks_mapper[dr.created.date()].append(dr)
+            else:
+                created_drinks_mapper[dr.created.date()] = [dr]
+
+        for created_date, d_records in created_drinks_mapper.items():
+            print(created_date)
+            print_days_purchases(money_spent_records, created_date)
+            print()
+            for index, d_record in enumerate(d_records):
+                print(f"({index}): ", d_record.category_item.name, get_created_tz(d_record.created))
+
+            for index, d_record in enumerate(d_records):
+                correct = input(f"Is ({index}) correct? y/n: ")
+                if correct.lower() == "y":
+                    correct_drink_ids.append(str(d_record.id))
+            print()
+            print("So far: ", correct_drink_ids)
+            print("===================================================================================================")
+
+
+def change_drinks_created():
+    with SessionLocal() as db_session:
+        record_repo = RecordRepository(db_session)
+        category_repo = CategoryRepository(db_session)
+
+        drinks_category = category_repo.get(name="Drinks")
+        drinks_records = (
+            db_session.query(RecordTable)
+            .join(RecordTable.category_item)
+            .filter(
+                RecordTable.id.notin_(CORRECT_RECORD_IDS),
+                RecordTable.category_item_id.in_([dc_item.id for dc_item in drinks_category.category_items]),
+            )
+            .order_by(RecordTable.created.asc())
+        )
+        for dr in drinks_records:
+            assert str(dr.id) not in CORRECT_RECORD_IDS, dr.id
+
+            dr.created = (dr.created - timedelta(days=1)).replace(hour=22, minute=0, second=0, microsecond=0)
+            record_repo.update(dr)
+        print(f"Updated {len(list(drinks_records))} records")

src/dusa_backend/scripts/remove_duplicate_stats.py

@@ -0,0 +1,27 @@
+from dusa_backend.domain.categories.repository import CategoryRepository
+from dusa_backend.domain.records.repository import RecordRepository
+from dusa_backend.infrastructure.database.session import SessionLocal
+from dusa_backend.infrastructure.database.tables import RecordTable
+
+
+def remove_duplicate_health_stats():
+    with SessionLocal() as db_session:
+        record_repo = RecordRepository(db_session)
+        category_repo = CategoryRepository(db_session)
+
+        health_category = category_repo.get(name="Health")
+        for category_item in health_category.category_items:
+            records = (
+                db_session.query(RecordTable)
+                .join(RecordTable.category_item)
+                .filter(RecordTable.category_item_id == category_item.id)
+                .order_by(RecordTable.value.desc())
+            )
+            dates_found = set()
+            delete_count = 0
+            for record in records:
+                if record.created.date() in dates_found:
+                    delete_count += record_repo.delete(id=record.id)
+                else:
+                    dates_found.add(record.created.date())
+            print(f"Deleted {delete_count} records for {category_item.name} category item")

src/dusa_backend/scripts/remove_records.py

@@ -0,0 +1,13 @@
+from dusa_backend.domain.records.repository import RecordRepository
+from dusa_backend.infrastructure.database.session import SessionLocal
+
+RECORD_IDS_TO_DELETE = ["59ee22e2-3e8f-4e4e-94b8-5f7bb78e1482", "147c52ba-e766-4c63-96b8-da51834f6528"]
+
+
+def remove_records():
+    count = 0
+    with SessionLocal() as db_session:
+        record_repo = RecordRepository(db_session)
+        for record_id in RECORD_IDS_TO_DELETE:
+            count += record_repo.delete(id=record_id)
+    print(f"Removed {count} records")