Refactors and adds comments to clr.py to explain the improvements

app/grants/clr_data_src.py

@@ -0,0 +1,150 @@
+from django.db import connection
+
+from grants.models import Contribution, Grant, GrantCollection
+from townsquare.models import SquelchProfile
+
+
+def fetch_grants(clr_round, network='mainnet'):
+    '''
+        Fetch grants that are included in the provided clr_round
+
+        Args:
+            network     :   mainnet | rinkeby
+            clr_round   :   GrantCLR
+        Returns:
+            grants      :   list of grants based on clr_type
+
+    '''
+
+    grant_filters = clr_round.grant_filters
+    collection_filters = clr_round.collection_filters
+
+    grants = clr_round.grants.filter(network=network, hidden=False, active=True, is_clr_eligible=True, link_to_new_grant=None)
+
+    if grant_filters:
+        # Grant Filters (grant_type, category)
+        grants = grants.filter(**grant_filters)
+    elif collection_filters:
+        # Collection Filters
+        grant_ids = GrantCollection.objects.filter(**collection_filters).values_list('grants', flat=True)
+        grants = grants.filter(pk__in=grant_ids)
+
+    return grants
+
+
+def fetch_contributions(clr_round, network='mainnet'):
+    '''
+        Fetch contributions that are included in the provided clr_round
+
+        Args:
+            network       :   mainnet | rinkeby
+            clr_round     :   GrantCLR
+        Returns:
+            contributions :   contributions data object
+
+    '''
+
+    clr_start_date = clr_round.start_date
+    clr_end_date = clr_round.end_date
+    grant_filters = clr_round.grant_filters
+    subscription_filters = clr_round.subscription_filters
+    collection_filters = clr_round.collection_filters
+
+    contributions = Contribution.objects.prefetch_related('subscription', 'profile_for_clr').filter(
+        match=True,
+        created_on__gte=clr_start_date,
+        created_on__lte=clr_end_date,
+        success=True,
+        subscription__network='mainnet'
+    ).nocache()
+
+    if subscription_filters:
+        contributions = contributions.filter(**subscription_filters)
+
+    # ignore profiles which have been squelched
+    profiles_to_be_ignored = SquelchProfile.objects.filter(active=True).values_list('profile__pk')
+    contributions = contributions.exclude(profile_for_clr__in=profiles_to_be_ignored)
+
+    return contributions
+
+
+def fetch_summed_contributions(grants, clr_round, network='mainnet'):
+    '''
+        Aggregated contributions grouped by grant and contributor
+
+        args:
+            grants      :   Grants (to fetch contribs for)
+            clr_round   :   GrantCLR
+            network     :   mainnet | rinkeby
+        returns:
+            aggregated contributions by pair nested dict
+                {
+                    grant_id (str): {
+                        user_id (str): aggregated_amount (float)
+                    }
+                }
+            dictionary of profile_ids and trust scores
+                {user_id (str): trust_score (float)}
+    '''
+
+    multiplier = clr_round.contribution_multiplier
+    clr_start_date = clr_round.start_date
+    clr_end_date = clr_round.end_date
+
+    # only consider contribs from current grant set
+    grantIds = ''
+    for i in range(len(grants)):
+        grantIds += "'" + str(grants[i].id) + "'" + (', ' if i+1 != len(grants) else '')
+
+    # collect contributions with a groupBy sum query
+    summedContribs = f'''
+        -- group by ... sum the contributions $ value for each user
+        SELECT
+            grants.use_grant_id as grant_id,
+            grants_contribution.profile_for_clr_id as user_id,
+            SUM((grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT * {float(multiplier)}),
+            MAX(dashboard_profile.as_dict ->> 'trust_bonus')::FLOAT as trust_bonus
+        FROM grants_contribution
+        INNER JOIN dashboard_profile ON (grants_contribution.profile_for_clr_id = dashboard_profile.id)
+        INNER JOIN grants_subscription ON (grants_contribution.subscription_id = grants_subscription.id)
+        RIGHT JOIN (
+            SELECT
+                grants_grant.id as grant_id,
+                (
+                    CASE
+                    WHEN grants_grant.defer_clr_to_id IS NOT NULL THEN grants_grant.defer_clr_to_id
+                    ELSE grants_grant.id
+                    END
+                ) as use_grant_id
+            FROM grants_grant
+        ) grants ON ((grants_contribution.normalized_data ->> 'id')::FLOAT = grants.grant_id)
+        WHERE (
+            grants_contribution.normalized_data ->> 'id' IN ({grantIds}) AND
+            grants_contribution.created_on >= '{clr_start_date}' AND
+            grants_contribution.created_on <= '{clr_end_date}' AND
+            grants_contribution.match = True AND
+            grants_subscription.network = '{network}' AND
+            grants_contribution.success = True AND
+            (grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT >= 0 AND
+            NOT (
+                grants_contribution.profile_for_clr_id IN (
+                    SELECT squelched.profile_id FROM townsquare_squelchprofile squelched WHERE squelched.active = True
+                ) AND grants_contribution.profile_for_clr_id IS NOT NULL
+            )
+        )
+        GROUP BY grants.use_grant_id, grants_contribution.profile_for_clr_id;
+    '''
+
+    # open cursor and execute the groupBy sum for the round
+    with connection.cursor() as cursor:
+        curr_agg = {}
+        trust_dict = {}
+        # execute to populate shared state for the round
+        cursor.execute(summedContribs)
+        for _row in cursor.fetchall():
+            if not curr_agg.get(_row[0]):
+                curr_agg[_row[0]] = {}
+            trust_dict[_row[1]] = _row[3]
+            curr_agg[_row[0]][_row[1]] = _row[2]
+
+    return curr_agg, trust_dict

app/grants/clr_old.py

@@ -22,8 +22,7 @@
 from django.utils import timezone
 
 import numpy as np
-from grants.models import Contribution, Grant, GrantCollection
-from townsquare.models import SquelchProfile
+from grants.clr_data_src import fetch_contributions, fetch_grants
 
 CLR_PERCENTAGE_DISTRIBUTED = 0
 
@@ -264,53 +263,6 @@ def calculate_clr_for_donation(grant, amount, grant_contributions_curr, total_po
     return (None, None, None, None)
 
 
-def fetch_data(clr_round, network='mainnet'):
-    '''
-        Populate Data needed to calculate CLR
-
-        Args:
-            network     :   mainnet | rinkeby
-            clr_round   :   GrantCLR
-        Returns:
-            contributions               : contributions data object
-            grants                      : list of grants based on clr_type
-
-    '''
-
-    clr_start_date = clr_round.start_date
-    clr_end_date = clr_round.end_date
-    grant_filters = clr_round.grant_filters
-    subscription_filters = clr_round.subscription_filters
-    collection_filters = clr_round.collection_filters
-
-    contributions = Contribution.objects.prefetch_related('subscription', 'profile_for_clr').filter(
-        match=True,
-        created_on__gte=clr_start_date,
-        created_on__lte=clr_end_date,
-        success=True,
-        subscription__network='mainnet'
-    ).nocache()
-
-    if subscription_filters:
-        contributions = contributions.filter(**subscription_filters)
-
-    # ignore profiles which have been squelched
-    profiles_to_be_ignored = SquelchProfile.objects.filter(active=True).values_list('profile__pk')
-    contributions = contributions.exclude(profile_for_clr__in=profiles_to_be_ignored)
-
-    grants = clr_round.grants.filter(network=network, hidden=False, active=True, is_clr_eligible=True, link_to_new_grant=None)
-
-    if grant_filters:
-        # Grant Filters (grant_type, category)
-        grants = grants.filter(**grant_filters)
-    elif collection_filters:
-        # Collection Filters
-        grant_ids = GrantCollection.objects.filter(**collection_filters).values_list('grants', flat=True)
-        grants = grants.filter(pk__in=grant_ids)
-
-    return grants, contributions
-
-
 def populate_data_for_clr(grants, contributions, clr_round):
     '''
         Populate Data needed to calculate CLR
@@ -402,8 +354,9 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
     v_threshold = float(clr_round.verified_threshold)
     uv_threshold = float(clr_round.unverified_threshold)
 
-    print(f"- starting fetch_data at {round(time.time(),1)}")
-    grants, contributions = fetch_data(clr_round, network)
+    print(f"- starting fetch_grants & fetch_contributions at {round(time.time(),1)}")
+    grants = fetch_grants(clr_round, network)
+    contributions = fetch_contributions(clr_round, network)
 
     if contributions.count() == 0:
         print(f'No Contributions for CLR {clr_round.round_num}. Exiting')
@@ -479,11 +432,10 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
                 potential_clr.append(predicted_clr)
 
         if save_to_db:
-            _grant = Grant.objects.get(pk=grant.pk)
             clr_prediction_curve = list(zip(potential_donations, potential_clr))
             base = clr_prediction_curve[0][1]
-            _grant.last_clr_calc_date = timezone.now()
-            _grant.next_clr_calc_date = timezone.now() + timezone.timedelta(minutes=60)
+            grant.last_clr_calc_date = timezone.now()
+            grant.next_clr_calc_date = timezone.now() + timezone.timedelta(minutes=60)
 
             can_estimate = True if base or clr_prediction_curve[1][1] or clr_prediction_curve[2][1] or clr_prediction_curve[3][1] else False
 
@@ -494,10 +446,10 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
 
             print(clr_prediction_curve)
 
-            clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve)
+            clr_round.record_clr_prediction_curve(grant, clr_prediction_curve)
 
             if from_date > (clr_calc_start_time - timezone.timedelta(hours=1)):
-                _grant.save()
+                grant.save()
 
         debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr})
 

app/grants/management/commands/analytics_clr.py

@@ -22,13 +22,13 @@
 from django.db import connection
 from django.utils import timezone
 
-from grants.clr import calculate_clr, fetch_grants, get_summed_contribs_query, get_totals_by_pair, normalise
+from grants.clr import calculate_clr, get_totals_by_pair, normalise
+from grants.clr_data_src import fetch_grants, fetch_summed_contributions
 from grants.models import GrantCLR
 
 
 def analytics_clr(from_date=None, clr_round=None, network='mainnet'):
     # setup
-    # clr_calc_start_time = timezone.now()
     debug_output = [['grant_id', 'grant_title', 'number_contributions', 'contribution_amount', 'clr_amount']]
 
     # one-time data call
@@ -37,40 +37,24 @@ def analytics_clr(from_date=None, clr_round=None, network='mainnet'):
 
     print(total_pot)
 
+    # fetch data
     grants = fetch_grants(clr_round, network)
-
-    # collect contributions for clr_round into temp table
-    initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network)
-
-    # open cursor and execute the groupBy sum for the round
-    with connection.cursor() as cursor:
-        curr_agg = {}
-        trust_dict = {}
-        # execute to populate shared state for the round
-        cursor.execute(initial_query) # (we could potential do better here by sharing this temp table between rounds)
-        for _row in cursor.fetchall():
-            if not curr_agg.get(_row[0]):
-                curr_agg[_row[0]] = {}
-
-            trust_dict[_row[1]] = _row[3]
-            curr_agg[_row[0]][_row[1]] = _row[2]
-
-        ptots = get_totals_by_pair(curr_agg)
-        bigtot, totals = calculate_clr(curr_agg, ptots, trust_dict, v_threshold, total_pot)
-
-        # normalise against a deepcopy of the totals to avoid mutations
-        curr_grants_clr = normalise(bigtot, totals, total_pot)
-
-        # calculate clr analytics output
-        for grant in grants:
-            num_contribs, contrib_amount, clr_amount = curr_grants_clr.get(grant.id, {'num_contribs': 0, 'contrib_amount': 0, 'clr_amount': None}).values()
-            # debug_output.append([grant.id, grant.title, num_contribs, contrib_amount, clr_amount])
-            debug_output.append([grant.id, grant.title, grant.positive_round_contributor_count, float(grant.amount_received_in_round), clr_amount])
+    curr_agg, trust_dict = fetch_summed_contributions(grants, clr_round, network)
+
+    # run calculation
+    ptots = get_totals_by_pair(curr_agg)
+    bigtot, totals = calculate_clr(curr_agg, ptots, trust_dict, v_threshold, total_pot)
+    curr_grants_clr = normalise(bigtot, totals, total_pot)
+
+    # calculate clr analytics output
+    for grant in grants:
+        num_contribs, contrib_amount, clr_amount = curr_grants_clr.get(grant.id, {'num_contribs': 0, 'contrib_amount': 0, 'clr_amount': None}).values()
+        # debug_output.append([grant.id, grant.title, num_contribs, contrib_amount, clr_amount])
+        debug_output.append([grant.id, grant.title, grant.positive_round_contributor_count, float(grant.amount_received_in_round), clr_amount])
 
     return debug_output
 
 
-
 class Command(BaseCommand):
 
     help = 'calculate clr base analytic results for all clr rounds or for a specific clr round'

app/grants/management/commands/analytics_clr_old.py

@@ -21,7 +21,8 @@
 from django.core.management.base import BaseCommand
 from django.utils import timezone
 
-from grants.clr_old import calculate_clr_for_donation, fetch_data, populate_data_for_clr
+from grants.clr_data_src import fetch_contributions, fetch_grants
+from grants.clr_old import calculate_clr_for_donation, populate_data_for_clr
 from grants.models import GrantCLR
 
 
@@ -37,7 +38,8 @@ def analytics_clr(from_date=None, clr_round=None, network='mainnet'):
 
     print(total_pot)
 
-    grants, contributions = fetch_data(clr_round, network)
+    grants = fetch_grants(clr_round, network)
+    contributions = fetch_contributions(clr_round, network)
 
     grant_contributions_curr = populate_data_for_clr(grants, contributions,  clr_round)
 

app/grants/router.py

@@ -3,6 +3,7 @@
 from django.core.paginator import EmptyPage, Paginator
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_control
+
 import django_filters.rest_framework
 from ratelimit.decorators import ratelimit
 from rest_framework import generics, mixins, routers, viewsets

app/grants/urls.py

@@ -24,11 +24,11 @@
     collection_thumbnail, contribute_to_grants_v1, contribution_addr_from_all_as_json,
     contribution_addr_from_grant_as_json, contribution_addr_from_grant_during_round_as_json,
     contribution_addr_from_round_as_json, contribution_info_from_grant_during_round_as_json, create_matching_pledge_v1,
-    flag, get_clr_sybil_input, get_collection, get_collections_list, get_ethereum_cart_data, get_grant_payload, get_grant_tags,
-    get_grants, get_interrupted_contributions, get_replaced_tx, get_trust_bonus, grant_activity, grant_details, grant_details_api,
-    grant_details_contributions, grant_details_contributors, grant_edit, grant_fund, grant_new, grants,
-    grants_addr_as_json, grants_bulk_add, grants_by_grant_type, grants_cart_view, grants_info, grants_landing,
-    grants_type_redirect, ingest_contributions, ingest_contributions_view, invoice, leaderboard,
+    flag, get_clr_sybil_input, get_collection, get_collections_list, get_ethereum_cart_data, get_grant_payload,
+    get_grant_tags, get_grants, get_interrupted_contributions, get_replaced_tx, get_trust_bonus, grant_activity,
+    grant_details, grant_details_api, grant_details_contributions, grant_details_contributors, grant_edit, grant_fund,
+    grant_new, grants, grants_addr_as_json, grants_bulk_add, grants_by_grant_type, grants_cart_view, grants_info,
+    grants_landing, grants_type_redirect, ingest_contributions, ingest_contributions_view, invoice, leaderboard,
     manage_ethereum_cart_data, new_matching_partner, profile, quickstart, remove_grant_from_collection, save_collection,
     toggle_grant_favorite, toggle_user_sybil, verify_grant,
 )

app/grants/views.py

@@ -69,7 +69,7 @@
 from economy.models import Token as FTokens
 from economy.utils import convert_token_to_usdt
 from eth_account.messages import defunct_hash_message
-from grants.clr_old import fetch_data
+from grants.clr_data_src import fetch_contributions
 from grants.models import (
     CartActivity, Contribution, Flag, Grant, GrantAPIKey, GrantBrandingRoutingPolicy, GrantCLR, GrantCollection,
     GrantTag, GrantType, MatchPledge, Subscription,
@@ -3587,7 +3587,7 @@ def get_clr_sybil_input(request, round_id):
         limit = data['limit'] if data['limit'] else 100
 
         # fetch grant contributions needed for round
-        __, all_clr_contributions = fetch_data(clr)
+        all_clr_contributions = fetch_contributions(clr)
         total_count = all_clr_contributions.count()
 
         # extract only needed fields