Add personal counts to DN aggregate routes metrics

packages/discovery-provider/ddl/migrations/0038_individual_routes_metrics.sql

@@ -0,0 +1,8 @@
+BEGIN;
+
+ALTER TABLE aggregate_daily_total_users_metrics ADD IF NOT EXISTS personal_count integer;
+ALTER TABLE aggregate_daily_unique_users_metrics ADD IF NOT EXISTS personal_count integer;
+ALTER TABLE aggregate_monthly_total_users_metrics ADD IF NOT EXISTS personal_count integer;
+ALTER TABLE aggregate_monthly_unique_users_metrics ADD IF NOT EXISTS personal_count integer;
+
+COMMIT;

packages/discovery-provider/integration_tests/queries/test_get_personal_route_metrics.py

@@ -0,0 +1,205 @@
+from datetime import date, timedelta
+
+from src.models.metrics.aggregate_daily_total_users_metrics import (
+    AggregateDailyTotalUsersMetrics,
+)
+from src.models.metrics.aggregate_daily_unique_users_metrics import (
+    AggregateDailyUniqueUsersMetrics,
+)
+from src.models.metrics.aggregate_monthly_total_users_metrics import (
+    AggregateMonthlyTotalUsersMetric,
+)
+from src.models.metrics.aggregate_monthly_unique_users_metrics import (
+    AggregateMonthlyUniqueUsersMetric,
+)
+from src.queries.get_personal_route_metrics import _get_personal_route_metrics
+from src.utils.db_session import get_db
+
+limit = 2
+today = date.today()
+yesterday = today - timedelta(days=1)
+
+
+def test_get_personal_route_metrics_week(app):
+    with app.app_context():
+        db_mock = get_db()
+
+    with db_mock.scoped_session() as session:
+        session.bulk_save_objects(
+            [
+                AggregateDailyUniqueUsersMetrics(
+                    count=1, personal_count=2, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=2, personal_count=1, timestamp=yesterday - timedelta(days=1)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=3, personal_count=3, timestamp=yesterday
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=4, personal_count=2, timestamp=today
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=2, personal_count=1, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=4, personal_count=4, timestamp=yesterday - timedelta(days=1)
+                ),
+                AggregateDailyTotalUsersMetrics(count=6, personal_count=5, timestamp=yesterday),
+                AggregateDailyTotalUsersMetrics(count=8, personal_count=8, timestamp=today),
+            ]
+        )
+
+        personal_metrics = _get_personal_route_metrics(session, "week", "day")
+
+        assert len(personal_metrics) == 2
+        assert personal_metrics[0]["unique_count"] == 1
+        assert personal_metrics[0]["total_count"] == 4
+        assert personal_metrics[1]["unique_count"] == 3
+        assert personal_metrics[1]["total_count"] == 5
+
+
+def test_get_personal_route_metrics_month_daily_bucket(app):
+    with app.app_context():
+        db_mock = get_db()
+
+    with db_mock.scoped_session() as session:
+        session.bulk_save_objects(
+            [
+                AggregateDailyUniqueUsersMetrics(
+                    count=1, personal_count=1, timestamp=today - timedelta(days=31)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=2, personal_count=1, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=3, personal_count=2, timestamp=yesterday
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=4, personal_count=5, timestamp=today
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=2, personal_count=2, timestamp=today - timedelta(days=31)
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=4, personal_count=3, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyTotalUsersMetrics(count=6, personal_count=6, timestamp=yesterday),
+                AggregateDailyTotalUsersMetrics(count=8, personal_count=5, timestamp=today),
+            ]
+        )
+
+        personal_metrics = _get_personal_route_metrics(session, "month", "day")
+
+        assert len(personal_metrics) == 2
+        assert personal_metrics[0]["unique_count"] == 1
+        assert personal_metrics[0]["total_count"] == 3
+        assert personal_metrics[1]["unique_count"] == 2
+        assert personal_metrics[1]["total_count"] == 6
+
+
+def test_get_personal_route_metrics_month_weekly_bucket(app):
+    with app.app_context():
+        db_mock = get_db()
+
+    with db_mock.scoped_session() as session:
+        session.bulk_save_objects(
+            [
+                AggregateDailyUniqueUsersMetrics(
+                    count=2, personal_count=1, timestamp=today - timedelta(days=31)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=2, personal_count=2, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=3, personal_count=3, timestamp=yesterday
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=4, personal_count=4, timestamp=today
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=2, personal_count=2, timestamp=today - timedelta(days=31)
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=4, personal_count=3, timestamp=today - timedelta(days=8)
+                ),
+                AggregateDailyTotalUsersMetrics(count=6, personal_count=6, timestamp=yesterday),
+                AggregateDailyTotalUsersMetrics(count=8, personal_count=7, timestamp=today),
+            ]
+        )
+
+        personal_metrics = _get_personal_route_metrics(session, "month", "week")
+
+        assert len(personal_metrics) == 2
+        assert personal_metrics[0]["unique_count"] == 2
+        assert personal_metrics[0]["total_count"] == 3
+        assert personal_metrics[1]["unique_count"] == 3
+        assert personal_metrics[1]["total_count"] == 6
+
+
+def test_get_personal_route_metrics_all_time_monthly_bucket(app):
+    with app.app_context():
+        db_mock = get_db()
+
+    with db_mock.scoped_session() as session:
+        session.bulk_save_objects(
+            [
+                AggregateMonthlyUniqueUsersMetric(
+                    count=1, personal_count=1, timestamp=today - timedelta(days=367)
+                ),
+                AggregateMonthlyUniqueUsersMetric(
+                    count=2, personal_count=2, timestamp=today - timedelta(days=100)
+                ),
+                AggregateMonthlyUniqueUsersMetric(
+                    count=3, personal_count=3, timestamp=today
+                ),
+                AggregateMonthlyTotalUsersMetric(
+                    count=2, personal_count=2, timestamp=today - timedelta(days=367)
+                ),
+                AggregateMonthlyTotalUsersMetric(
+                    count=4, personal_count=3, timestamp=today - timedelta(days=100)
+                ),
+                AggregateMonthlyTotalUsersMetric(count=6, personal_count=4, timestamp=today),
+            ]
+        )
+
+        personal_metrics = _get_personal_route_metrics(session, "all_time", "month")
+
+        assert len(personal_metrics) == 2
+        assert personal_metrics[0]["unique_count"] == 1
+        assert personal_metrics[0]["total_count"] == 2
+        assert personal_metrics[1]["unique_count"] == 2
+        assert personal_metrics[1]["total_count"] == 3
+
+
+def test_get_personal_route_metrics_all_time_weekly_bucket(app):
+    with app.app_context():
+        db_mock = get_db()
+
+    with db_mock.scoped_session() as session:
+        session.bulk_save_objects(
+            [
+                AggregateDailyUniqueUsersMetrics(
+                    count=1, personal_count=1, timestamp=today - timedelta(days=367)
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=2, personal_count=2, timestamp=yesterday
+                ),
+                AggregateDailyUniqueUsersMetrics(
+                    count=3, personal_count=3, timestamp=today
+                ),
+                AggregateDailyTotalUsersMetrics(
+                    count=2, personal_count=1, timestamp=today - timedelta(days=367)
+                ),
+                AggregateDailyTotalUsersMetrics(count=4, personal_count=3, timestamp=yesterday),
+                AggregateDailyTotalUsersMetrics(count=6, personal_count=6, timestamp=today),
+            ]
+        )
+
+        personal_metrics = _get_personal_route_metrics(session, "all_time", "week")
+
+        assert len(personal_metrics) == 2
+        assert personal_metrics[0]["unique_count"] == 1
+        assert personal_metrics[0]["total_count"] == 1
+        assert personal_metrics[1]["unique_count"] == 2
+        assert personal_metrics[1]["total_count"] == 3

packages/discovery-provider/integration_tests/tasks/test_index_trending_notification.py

@@ -36,7 +36,9 @@ def test_index_trending_notification(app):
         trending_strategy = trending_strategy_factory.get_strategy(TrendingType.TRACKS)
         # Test that if there are no prev notifications, all trending in top are generated
 
-        trending_key = make_trending_tracks_cache_key("week", None, trending_strategy.version)
+        trending_key = make_trending_tracks_cache_key(
+            "week", None, trending_strategy.version
+        )
         trending_tracks = [
             {
                 "track_id": i,

packages/discovery-provider/src/api/v1/metrics.py

@@ -19,6 +19,7 @@
 )
 from src.queries.get_genre_metrics import get_genre_metrics
 from src.queries.get_historical_route_metrics import get_historical_route_metrics
+from src.queries.get_personal_route_metrics import get_personal_route_metrics
 from src.queries.get_plays_metrics import get_plays_metrics
 from src.queries.get_trailing_metrics import get_aggregate_route_metrics_trailing_month
 from src.utils.redis_cache import cache
@@ -69,6 +70,37 @@
 }
 
 
+personal_route_metrics_parser = reqparse.RequestParser()
+personal_route_metrics_parser.add_argument("bucket_size", required=False)
+
+
+@ns.route("/routes/<string:time_range>", doc=False)
+class PersonalRouteMetrics(Resource):
+    @ns.doc(
+        id="""Personal Route Metrics""",
+        params={
+            "bucket_size": "Grouping of route metrics (e.g. by day, week, or month) for given time range"
+        },
+        responses={200: "Success", 400: "Bad request", 500: "Server error"},
+    )
+    @cache(ttl_sec=30 * 60)
+    def get(self, time_range):
+        """Gets node's personal route metrics based on time range and bucket size"""
+        if time_range not in valid_bucket_sizes:
+            abort_bad_path_param("time_range", ns)
+
+        args = personal_route_metrics_parser.parse_args()
+        valid_buckets = valid_bucket_sizes[time_range]
+        bucket_size = args.get("bucket_size") or valid_buckets[0]
+
+        if bucket_size not in valid_buckets:
+            abort_bad_request_param("bucket_size", ns)
+
+        metrics = get_personal_route_metrics(time_range, bucket_size)
+        response = success_response(metrics)
+        return response
+
+
 @ns.route("/routes/cached", doc=False)
 class CachedRouteMetrics(Resource):
     @ns.doc(

packages/discovery-provider/src/models/delisting/track_delist_status.py

@@ -9,10 +9,10 @@
 class DelistTrackReason(str, enum.Enum):
     DMCA = "DMCA"
     ACR = "ACR"
-    MANUAL = "MANUAL",
-    ACR_COUNTER_NOTICE = "ACR_COUNTER_NOTICE",
-    DMCA_RETRACTION = "DMCA_RETRACTION",
-    DMCA_COUNTER_NOTICE = "DMCA_COUNTER_NOTICE",
+    MANUAL = "MANUAL"
+    ACR_COUNTER_NOTICE = "ACR_COUNTER_NOTICE"
+    DMCA_RETRACTION = "DMCA_RETRACTION"
+    DMCA_COUNTER_NOTICE = "DMCA_COUNTER_NOTICE"
     DMCA_AND_ACR_COUNTER_NOTICE = "DMCA_AND_ACR_COUNTER_NOTICE"
 
 

packages/discovery-provider/src/models/metrics/aggregate_daily_total_users_metrics.py

@@ -19,3 +19,4 @@ class AggregateDailyTotalUsersMetrics(Base, RepresentableMixin):
     updated_at = Column(
         DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
     )
+    personal_count = Column(Integer)

packages/discovery-provider/src/models/metrics/aggregate_daily_unique_users_metrics.py

@@ -20,3 +20,4 @@ class AggregateDailyUniqueUsersMetrics(Base, RepresentableMixin):
         DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
     )
     summed_count = Column(Integer)
+    personal_count = Column(Integer)

packages/discovery-provider/src/models/metrics/aggregate_monthly_total_users_metrics.py

@@ -19,3 +19,4 @@ class AggregateMonthlyTotalUsersMetric(Base, RepresentableMixin):
     updated_at = Column(
         DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
     )
+    personal_count = Column(Integer)

packages/discovery-provider/src/models/metrics/aggregate_monthly_unique_users_metrics.py

@@ -20,3 +20,4 @@ class AggregateMonthlyUniqueUsersMetric(Base, RepresentableMixin):
         DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP")
     )
     summed_count = Column(Integer)
+    personal_count = Column(Integer)

packages/discovery-provider/src/queries/get_aggregate_route_metrics.py

@@ -24,7 +24,7 @@
 
 def get_aggregate_route_metrics(time_range, bucket_size):
     """
-    Returns a list of timestamp with unique count and total count for all routes
+    Returns a list of timestamps with unique count and total count for all routes
     based on given time range and grouped by bucket size
 
     Returns: