Migrate release_date from string to date (#6655)

AudiusProject · Nov 14, 2023 · 9a83de5 · 9a83de5
1 parent c7445f9
commit 9a83de5
Show file tree

Hide file tree

Showing 13 changed files with 107 additions and 91 deletions.
diff --git a/packages/discovery-provider/ddl/migrations/0039_migrate_date.sql b/packages/discovery-provider/ddl/migrations/0039_migrate_date.sql
@@ -0,0 +1,22 @@
+-- migrate release_date to a timestamp
+begin;
+lock table tracks in access exclusive mode;
+
+alter table tracks disable trigger on_track;
+alter table tracks disable trigger trg_tracks;
+
+alter table tracks add column release_date_temp timestamp;
+
+update tracks set release_date_temp = to_timestamp(release_date, 'YYYY-MM-DD"T"HH24:MI:SS.US')::timestamp
+where release_date like '%Z';
+
+update tracks set release_date_temp = to_timestamp(substring(release_date from 1 for 20), 'Dy Mon DD YYYY HH24:MI:SS')
+where release_date like '%GMT%' and length(release_date) > 22;
+
+alter table tracks drop column release_date;
+
+alter table tracks rename column release_date_temp to release_date;
+
+alter table tracks enable trigger on_track;
+alter table tracks enable trigger trg_tracks;
+commit;
diff --git a/packages/discovery-provider/integration_tests/queries/test_get_playlists.py b/packages/discovery-provider/integration_tests/queries/test_get_playlists.py
@@ -88,7 +88,7 @@ def test_entities():
                 "title": "track 1",
                 "owner_id": 1287289,
                 "is_current": True,
-                "release_date": "Fri Dec 20 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 20),
                 "created_at": datetime(2018, 5, 17),
             },
             {
@@ -103,7 +103,7 @@ def test_entities():
                 "title": "track 3",
                 "owner_id": 1287289,
                 "is_current": True,
-                "release_date": "Wed Dec 18 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 18),
                 "created_at": datetime(2020, 5, 17),
                 "is_unlisted": True,
             },

diff --git a/packages/discovery-provider/integration_tests/queries/test_get_track_library.py b/packages/discovery-provider/integration_tests/queries/test_get_track_library.py
@@ -17,7 +17,7 @@ def populate_tracks(db):
                 "track_id": 17,
                 "title": "track 17",
                 "owner_id": 1287290,
-                "release_date": "Mon Jan 20 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 17),
                 "created_at": datetime(2019, 6, 17),
             },
             {
@@ -30,58 +30,58 @@ def populate_tracks(db):
                 "track_id": 19,
                 "title": "track 19",
                 "owner_id": 1287290,
-                "release_date": "Tue Jan 21 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2021, 6, 17),
                 "ai_attribut /duion_user_id": 1287290,
             },
             {
                 "track_id": 20,
                 "title": "a 20",
                 "owner_id": 1287290,
-                "release_date": "Wed Jan 22 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 19),
             },
             {
                 "track_id": 21,
                 "title": "z 21",
                 "owner_id": 6,
-                "release_date": "Tue Jan 21 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 20),
             },
             {
                 "track_id": 22,
                 "title": "track 22",
                 "owner_id": 5,
-                "release_date": "Tue Jan 21 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2021, 6, 17),
             },
             {
                 "track_id": 23,
                 "title": "some_title",
                 "owner_id": 7,
-                "release_date": "Sun Jan 19 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 16),
             },
             {
                 "track_id": 24,
                 "title": "track 24",
                 "owner_id": 5,
-                "release_date": "Wed Jan 22 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 20),
             },
             {
                 "track_id": 25,
                 "title": "track 25",
                 "owner_id": 5,
-                "release_date": "Wed Jan 22 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 20),
                 "is_unlisted": True,
             },
             {
                 "track_id": 26,
                 "title": "track 26",
                 "owner_id": 5,
-                "release_date": "Wed Jan 22 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 6, 18),
                 "created_at": datetime(2019, 6, 21),
                 "is_delete": True,
             },

diff --git a/packages/discovery-provider/integration_tests/queries/test_get_tracks.py b/packages/discovery-provider/integration_tests/queries/test_get_tracks.py
@@ -14,7 +14,7 @@ def populate_tracks(db):
                 "track_id": 1,
                 "title": "track 1",
                 "owner_id": 1287289,
-                "release_date": "Fri Dec 20 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 20),
                 "created_at": datetime(2018, 5, 17),
             },
             {
@@ -27,104 +27,104 @@ def populate_tracks(db):
                 "track_id": 3,
                 "title": "track 3",
                 "owner_id": 1287289,
-                "release_date": "Wed Dec 18 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 18),
                 "created_at": datetime(2020, 5, 17),
                 "ai_attribution_user_id": 1287289,
             },
             {
                 "track_id": 4,
                 "title": "track 4",
                 "owner_id": 1287289,
-                "release_date": "",
+                "release_date": None,
                 "created_at": datetime(2018, 5, 19),
             },
             {
                 "track_id": 5,
                 "title": "track 5",
                 "owner_id": 1287289,
-                "release_date": "garbage-should-not-parse",
+                "release_date": None,
                 "created_at": datetime(2018, 5, 20),
             },
             {
                 "track_id": 6,
                 "title": "track 6",
                 "owner_id": 4,
-                "release_date": "Wed Dec 18 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 18),
                 "created_at": datetime(2020, 5, 17),
             },
             {
                 "track_id": 7,
                 "title": "track 7",
                 "owner_id": 4,
-                "release_date": "",
+                "release_date": None,
                 "created_at": datetime(2018, 5, 19),
             },
             {
                 "track_id": 8,
                 "title": "track 8",
                 "owner_id": 4,
-                "release_date": "garbage-should-not-parse",
+                "release_date": None,
                 "created_at": datetime(2018, 5, 20),
             },
             {
                 "track_id": 9,
                 "title": "track 9",
                 "owner_id": 4,
-                "release_date": "Wed Dec 25 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 25),
                 "created_at": datetime(2018, 5, 20),
                 "is_unlisted": True,
             },
             {
                 "track_id": 10,
                 "title": "track 10",
                 "owner_id": 4,
-                "release_date": "Wed Dec 25 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 25),
                 "created_at": datetime(2018, 5, 21),
                 "is_delete": True,
             },
             {
                 "track_id": 11,
                 "title": "track 11",
                 "owner_id": 1287289,
-                "release_date": "Fri Dec 19 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 19),
                 "created_at": datetime(2018, 5, 17),
                 "is_unlisted": True,
             },
             {
                 "track_id": 12,
                 "title": "track 12",
                 "owner_id": 5,
-                "release_date": "Fri Jun 19 2020 12:00:00 GMT-0800",
+                "release_date": datetime(2020, 6, 19),
                 "created_at": datetime(2018, 5, 21),
                 "ai_attribution_user_id": 1287289,
             },
             {
                 "track_id": 13,
                 "title": "track 13",
                 "owner_id": 5,
-                "release_date": "Fri Oct 7 2022 12:00:00 GMT-0800",
+                "release_date": datetime(2022, 10, 7),
                 "created_at": datetime(2018, 5, 17),
             },
             {
                 "track_id": 14,
                 "title": "track 14",
                 "owner_id": 5,
-                "release_date": "Wed Dec 25 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 25),
                 "created_at": datetime(2020, 5, 17),
             },
             {
                 "track_id": 15,
                 "title": "track 15",
                 "owner_id": 1287289,
-                "release_date": "",
-                "created_at": datetime(2017, 5, 19, 1),
+                "release_date": None,
+                "created_at": datetime(2017, 5, 19),
             },
             {
                 "track_id": 16,
                 "title": "track 16",
                 "owner_id": 1287289,
-                "release_date": "Fri May 19 2017 12:00:00 GMT-0800",
-                "created_at": datetime(2017, 5, 19, 10),
+                "release_date": datetime(2017, 5, 19),
+                "created_at": datetime(2017, 5, 19),
             },
         ],
         "track_routes": [
@@ -188,8 +188,8 @@ def test_get_tracks_by_date(app):
 
         # tracks created on the same day, with one missing 'release_date`
         # should fall back to sorting by id
-        assert tracks[5]["track_id"] == 16
-        assert tracks[6]["track_id"] == 15
+        assert tracks[5]["track_id"] == 15
+        assert tracks[6]["track_id"] == 16
 
         assert tracks[0]["permalink"] == "/some-test-user/track-1"
         assert tracks[4]["permalink"] == "/some-test-user/track-2"

diff --git a/...es/discovery-provider/integration_tests/tasks/entity_manager/test_track_entity_manager.py b/...es/discovery-provider/integration_tests/tasks/entity_manager/test_track_entity_manager.py
@@ -48,7 +48,7 @@ def test_index_valid_track(app, mocker):
             "created_at": "2020-07-11 08:22:15",
             "create_date": None,
             "updated_at": "2020-07-11 08:22:15",
-            "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700",
+            "release_date": "2020-07-11 08:22:15",
             "file_type": None,
             "is_playlist_upload": True,
             "duration": 100,
@@ -474,7 +474,7 @@ def test_index_invalid_tracks(app, mocker):
             "created_at": "2020-07-11 08:22:15",
             "create_date": None,
             "updated_at": "2020-07-11 08:22:15",
-            "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700",
+            "release_date": "2020-07-11 08:22:15",
             "file_type": None,
             "track_segments": [
                 {
@@ -916,7 +916,7 @@ def test_invalid_track_description(app, mocker):
             "created_at": "2020-07-11 08:22:15",
             "create_date": None,
             "updated_at": "2020-07-11 08:22:15",
-            "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700",
+            "release_date": "2020-07-11 08:22:15",
             "file_type": None,
             "is_playlist_upload": True,
             "track_segments": [

diff --git a/...ges/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py b/...ges/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py
@@ -379,7 +379,7 @@ def get_events_side_effect(_, tx_receipt):
                 "track_id": TRACK_ID_OFFSET,
                 "title": "track 1",
                 "owner_id": USER_ID_OFFSET,
-                "release_date": "Fri Dec 20 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 20),
                 "created_at": datetime(2018, 5, 17),
             }
         ],
@@ -805,7 +805,7 @@ def get_events_side_effect(_, tx_receipt):
                 "track_id": TRACK_ID_OFFSET,
                 "title": "track 1",
                 "owner_id": USER_ID_OFFSET,
-                "release_date": "Fri Dec 20 2019 12:00:00 GMT-0800",
+                "release_date": datetime(2019, 12, 20),
                 "created_at": datetime(2018, 5, 17),
             }
         ],

diff --git a/packages/discovery-provider/integration_tests/utils.py b/packages/discovery-provider/integration_tests/utils.py
@@ -213,7 +213,7 @@ def populate_mock_db(db, entities, block_offset=None):
                 remix_of=track_meta.get("remix_of", None),
                 updated_at=track_meta.get("updated_at", track_created_at),
                 created_at=track_meta.get("created_at", track_created_at),
-                release_date=track_meta.get("release_date", None),
+                release_date=str(track_meta.get("release_date")) if track_meta.get("release_date") else None,
                 is_unlisted=track_meta.get("is_unlisted", False),
                 is_premium=track_meta.get("is_premium", False),
                 premium_conditions=track_meta.get("premium_conditions", None),

diff --git a/packages/discovery-provider/src/models/tracks/track.py b/packages/discovery-provider/src/models/tracks/track.py
@@ -49,7 +49,7 @@ class Track(Base, RepresentableMixin):
     mood = Column(String)
     credits_splits = Column(String)
     create_date = Column(String)
-    release_date = Column(String)
+    release_date = Column(DateTime)
     file_type = Column(String)
     metadata_multihash = Column(String)
     track_segments = Column(JSONB(), nullable=False)

diff --git a/packages/discovery-provider/src/queries/get_track_library.py b/packages/discovery-provider/src/queries/get_track_library.py
@@ -1,6 +1,6 @@
 from typing import Optional, TypedDict
 
-from sqlalchemy import asc, desc, func, or_
+from sqlalchemy import asc, desc, or_
 from sqlalchemy.orm import contains_eager
 from sqlalchemy.sql.functions import coalesce, max
 
@@ -175,10 +175,7 @@ def _get_track_library(args: GetTrackLibraryArgs, session):
         base_query = base_query.order_by(
             sort_fn(
                 coalesce(
-                    func.to_date_safe(
-                        TrackWithAggregates.release_date,
-                        "Dy Mon DD YYYY HH24:MI:SS",
-                    ),
+                    TrackWithAggregates.release_date,
                     TrackWithAggregates.created_at,
                 )
             )

diff --git a/packages/discovery-provider/src/queries/get_tracks.py b/packages/discovery-provider/src/queries/get_tracks.py
@@ -1,7 +1,7 @@
 import logging  # pylint: disable=C0302
 from typing import List, Optional, TypedDict
 
-from sqlalchemy import and_, asc, case, desc, func, or_
+from sqlalchemy import and_, asc, case, desc, or_
 from sqlalchemy.sql.functions import coalesce
 
 from src.models.social.aggregate_plays import AggregatePlay
@@ -170,10 +170,7 @@ def _get_tracks(session, args):
             base_query = base_query.order_by(
                 sort_fn(
                     coalesce(
-                        func.to_timestamp_safe(
-                            TrackWithAggregates.release_date,
-                            "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM",
-                        ),
+                        TrackWithAggregates.release_date,
                         TrackWithAggregates.created_at,
                     )
                 ),
@@ -213,10 +210,7 @@ def _get_tracks(session, args):
             base_query = base_query.order_by(
                 coalesce(
                     # This func is defined in migrations
-                    func.to_timestamp_safe(
-                        TrackWithAggregates.release_date,
-                        "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM",
-                    ),
+                    TrackWithAggregates.release_date,
                     TrackWithAggregates.created_at,
                 ).desc(),
                 TrackWithAggregates.track_id,

diff --git a/packages/discovery-provider/src/queries/get_user_listening_history.py b/packages/discovery-provider/src/queries/get_user_listening_history.py
@@ -153,9 +153,7 @@ def sort_by_sort_method(sort_method, sort_fn, track_ids, base_query):
         return base_query.order_by(
             sort_fn(
                 coalesce(
-                    func.to_date_safe(
-                        TrackWithAggregates.release_date, "Dy Mon DD YYYY HH24:MI:SS"
-                    ),
+                    TrackWithAggregates.release_date,
                     TrackWithAggregates.created_at,
                 )
             )