diff --git a/app/dashboard/management/commands/update_trust_bonus.py b/app/dashboard/management/commands/update_trust_bonus.py new file mode 100644 index 00000000000..3baf483c972 --- /dev/null +++ b/app/dashboard/management/commands/update_trust_bonus.py @@ -0,0 +1,46 @@ +''' + Copyright (C) 2021 Gitcoin Core + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +''' + +from django.core.management.base import BaseCommand + +from dashboard.models import Profile +from dashboard.tasks import update_trust_bonus + + +class Command(BaseCommand): + + help = 'Update every users trust_bonus score' + + def add_arguments(self, parser): + parser.add_argument( + '--call-now', + type=int, + help="disable execution on celery and call now" + ) + + def handle(self, *args, **options): + profiles = Profile.objects.all() + print(profiles.count()) + for profile in profiles.iterator(): + if (options['call_now']): + params = profile.as_dict + params['trust_bonus'] = profile.trust_bonus + print("Saving - %s - %s" % (profile.handle, params['trust_bonus'])) + profile.save() + else: + update_trust_bonus.delay(profile.pk) diff --git a/app/dashboard/models.py b/app/dashboard/models.py index 82517d09cad..e8f4f75743a 100644 --- a/app/dashboard/models.py +++ b/app/dashboard/models.py @@ -4400,6 +4400,7 @@ def to_dict(self): 'card_title': f'@{self.handle} | Gitcoin', 'org_works_with': org_works_with, 'card_desc': desc, + 'trust_bonus': self.trust_bonus, 'avatar_url': self.avatar_url_with_gitcoin_logo, 'count_bounties_completed': total_fulfilled, 'works_with_collected': works_with_collected, diff --git a/app/dashboard/tasks.py b/app/dashboard/tasks.py index f0eb2901467..3cd4ffe69b5 100644 --- a/app/dashboard/tasks.py +++ b/app/dashboard/tasks.py @@ -246,6 +246,21 @@ def profile_dict(self, pk, retry: bool = True) -> None: profile.save() +@app.shared_task(bind=True, max_retries=3) +def update_trust_bonus(self, pk): + """ + :param self: + :param pk: + :return: + """ + profile = Profile.objects.get(pk=pk) + params = profile.as_dict + if profile.trust_bonus != params.get('trust_bonus', None): + params['trust_bonus'] = profile.trust_bonus + print("Saving - %s - %s" % (profile.handle, params['trust_bonus'])) + profile.save() + + @app.shared_task(bind=True) def maybe_market_to_user_slack(self, bounty_pk, event_name, retry: bool = True) -> None: """ diff --git a/app/grants/admin.py b/app/grants/admin.py index 76d594efde7..e49ac11b4e6 100644 --- a/app/grants/admin.py +++ b/app/grants/admin.py @@ -104,7 +104,7 @@ class GrantAdmin(GeneralAdmin): 'active', 'visible', 'is_clr_eligible', 'migrated_to', 'region', 'grant_type', 'categories', 'description', 'description_rich', 'github_project_url', 'reference_url', 'admin_address', - 'amount_received', 'amount_received_in_round', 'monthly_amount_subscribed', + 'amount_received', 'amount_received_in_round', 'monthly_amount_subscribed', 'defer_clr_to', 'deploy_tx_id', 'cancel_tx_id', 'admin_profile', 'token_symbol', 'token_address', 'contract_address', 'contract_version', 'network', 'required_gas_price', 'logo_svg_asset', 'logo_asset', 'created_on', 'modified_on', 'team_member_list', diff --git a/app/grants/clr.py b/app/grants/clr.py index 5be90eb2fa8..d7968bea8fc 100644 --- a/app/grants/clr.py +++ b/app/grants/clr.py @@ -415,7 +415,12 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn print(f"- starting slim grant calc at {round(time.time(),1)}") grants_clr = run_clr_calcs(grant_contributions_curr, v_threshold, uv_threshold, total_pot) print(f"- saving slim grant calc at {round(time.time(),1)}") + total_count = len(grants_clr.items()) for grant_calc in grants_clr: + counter += 1 + if counter % 10 == 0 or True: + print(f"- {counter}/{total_count} grants iter, pk:{grant_id}, at {round(time.time(),1)}") + pk = grant_calc['id'] grant = clr_round.grants.using('default').get(pk=pk) latest_calc = grant.clr_calculations.using('default').filter(latest=True, grantclr=clr_round).order_by('-pk').first() @@ -424,10 +429,12 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn continue clr_prediction_curve = copy.deepcopy(latest_calc.clr_prediction_curve) clr_prediction_curve[0][1] = grant_calc['clr_amount'] # update only the existing match estimate + print(clr_prediction_curve) clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) grant.save() # if we are only calculating slim CLR calculations, return here and save 97% compute power print(f"- done calculating at {round(time.time(),1)}") + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}") return print(f"- starting grants iter at {round(time.time(),1)}") @@ -460,7 +467,6 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn else: for amount in potential_donations: # calculate clr with each additional donation and save to grants model - # print(f'using {total_pot_close}') predicted_clr, grants_clr, _, _ = calculate_clr_for_donation( grant, amount, grant_contributions_curr, total_pot, v_threshold, uv_threshold ) @@ -480,6 +486,8 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn else: clr_prediction_curve = [[0.0, 0.0, 0.0] for x in range(0, 6)] + print(clr_prediction_curve) + clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) if from_date > (clr_calc_start_time - timezone.timedelta(hours=1)): @@ -487,4 +495,6 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}") + return debug_output diff --git a/app/grants/clr2.py b/app/grants/clr2.py new file mode 100644 index 00000000000..7a9bf054945 --- /dev/null +++ b/app/grants/clr2.py @@ -0,0 +1,330 @@ +# -*- coding: utf-8 -*- +"""Define the Grants application configuration. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" +import copy + +from django.db import connection +from django.utils import timezone + +import numpy as np +from dashboard.models import Profile +from grants.models import Grant, GrantCollection +from townsquare.models import SquelchProfile + +CLR_PERCENTAGE_DISTRIBUTED = 0 + + +def get_summed_contribs_query(grants, created_after, created_before, multiplier, network): + # only consider contribs from current grant set + grantIds = '' + for i in range(len(grants)): + grantIds += "'" + str(grants[i].id) + "'" + (', ' if i+1 != len(grants) else '') + + summedContribs = f''' + -- drop the table if it exists + DROP TABLE IF EXISTS tempUserTotals; + + -- group by ... sum the contributions $ value for each user + SELECT + grants.use_grant_id as grant_id, + grants_contribution.profile_for_clr_id as user_id, + SUM((grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT * {float(multiplier)}), + MAX(dashboard_profile.as_dict ->> 'trust_bonus')::FLOAT as trust_bonus + INTO TEMP TABLE tempUserTotals + FROM grants_contribution + INNER JOIN dashboard_profile ON (grants_contribution.profile_for_clr_id = dashboard_profile.id) + INNER JOIN grants_subscription ON (grants_contribution.subscription_id = grants_subscription.id) + RIGHT JOIN ( + SELECT + grants_grant.id as grant_id, + ( + CASE + WHEN grants_grant.defer_clr_to_id IS NOT NULL THEN grants_grant.defer_clr_to_id + ELSE grants_grant.id + END + ) as use_grant_id + FROM grants_grant + ) grants ON ((grants_contribution.normalized_data ->> 'id')::FLOAT = grants.grant_id) + WHERE ( + grants_contribution.normalized_data ->> 'id' IN ({grantIds}) AND + grants_contribution.created_on >= '{created_after}' AND + grants_contribution.created_on <= '{created_before}' AND + grants_contribution.match = True AND + grants_subscription.network = '{network}' AND + grants_contribution.success = True AND + (grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT >= 0 AND + NOT ( + grants_contribution.profile_for_clr_id IN ( + SELECT squelched.profile_id FROM townsquare_squelchprofile squelched WHERE squelched.active = True + ) AND grants_contribution.profile_for_clr_id IS NOT NULL + ) + ) + GROUP BY grants.use_grant_id, grants_contribution.profile_for_clr_id; + + -- index before joining in clr_query + CREATE INDEX ON tempUserTotals (grant_id, user_id); + + SELECT * FROM tempUserTotals; + ''' + + return summedContribs + + +def add_prediction_contrib_query(grant_id, amount): + predictionContrib = f''' + -- delete any previous prediction values from the contributions table + DELETE FROM tempUserTotals WHERE user_id = 999999999; + + -- insert the prediction value into contributions (grant_id, user_id, amount, trust_bonus) + {"INSERT INTO tempUserTotals VALUES(" + str(grant_id) + ", 999999999, " + str(amount) + ", 1);" if amount != 0 else ""} + ''' + return predictionContrib + + +def get_calc_query(v_threshold): + pairwise = ''' + -- produce the pairwise sums + SELECT + c1.user_id, + c2.user_id as user_id_2, + SUM((c1.sum * c2.sum) ^ 0.5) pairwise + FROM tempUserTotals c1 + INNER JOIN tempUserTotals c2 ON (c1.grant_id = c2.grant_id AND c2.user_id > c1.user_id) + GROUP BY c1.user_id, c2.user_id + ''' + + clrAmount = f''' + -- calculate the CLR amount for each grant + SELECT + c1.grant_id, + -- add trust scores and threshold here + SUM((c1.sum * c2.sum) ^ 0.5 / (pw.pairwise / ({v_threshold} * GREATEST(c2.trust_bonus, c1.trust_bonus)) + 1)) final_clr + FROM tempUserTotals c1 + INNER JOIN tempUserTotals c2 ON (c1.grant_id = c2.grant_id AND c2.user_id > c1.user_id) + INNER JOIN ({pairwise}) pw ON (c1.user_id = pw.user_id AND c2.user_id = pw.user_id_2) + GROUP BY c1.grant_id + ORDER BY c1.grant_id; + ''' + + # # CTE of pairwise, clrAmount and clrResult (this will return the clr_amount, number_contribtions and contribution_amount for each grant) + # clrResult = f''' + # -- group by ... sum the contributions $ value for each grant and place the clr + # SELECT + # c1.grant_id, + # -- use MAX/MIN because we know we will only match a single CLR here + # MAX(clr.final_clr) clr_amount, + # SUM(1) number_contributions, + # SUM(c1.sum) contribution_amount + # FROM tempUserTotals c1 + # INNER JOIN ({clrAmount}) clr ON (c1.grant_id = clr.grant_id) + # GROUP BY c1.grant_id; + # ''' + + return clrAmount + + +def fetch_grants(clr_round, network='mainnet'): + grant_filters = clr_round.grant_filters + collection_filters = clr_round.collection_filters + + grants = clr_round.grants.filter(network=network, hidden=False, active=True, is_clr_eligible=True, link_to_new_grant=None) + + if grant_filters: + # Grant Filters (grant_type, category) + grants = grants.filter(**grant_filters) + elif collection_filters: + # Collection Filters + grant_ids = GrantCollection.objects.filter(**collection_filters).values_list('grants', flat=True) + grants = grants.filter(pk__in=grant_ids) + + return grants + + +def calculate_clr_for_donation(grant_id, amount, cursor, total_pot, v_threshold): + # collect results + bigtot = 0 + totals = {} + + # find grant in contributions list and add donation + clr_query = add_prediction_contrib_query(grant_id, amount) + get_calc_query(v_threshold) + cursor.execute(clr_query) + for _row in cursor.fetchall(): + bigtot += _row[1] if _row[1] else 0 + totals[_row[0]] = {'clr_amount': _row[1]} + + global CLR_PERCENTAGE_DISTRIBUTED + + # check if saturation is reached + if bigtot >= total_pot: # saturation reached + # print(f'saturation reached. Total Pot: ${total_pot} | Total Allocated ${bigtot}. Normalizing') + CLR_PERCENTAGE_DISTRIBUTED = 100 + for pk, grant_calc in totals.items(): + grant_calc['clr_amount'] = ((grant_calc['clr_amount'] / bigtot) * total_pot) + else: + CLR_PERCENTAGE_DISTRIBUTED = (bigtot / total_pot) * 100 + if bigtot == 0: + bigtot = 1 + percentage_increase = np.log(total_pot / bigtot) / 100 + for pk, grant_calc in totals.items(): + grant_calc['clr_amount'] = grant_calc['clr_amount'] * (1 + percentage_increase) if grant_calc['clr_amount'] else 0 + + # find grant we added the contribution to and get the new clr amount + if grant_id and totals.get(grant_id): + clr = totals[grant_id] + return ( + clr, + clr['clr_amount'] + ) + + # print(f'info: no contributions found for grant {grant_id}') + return (totals, 0.0) + + +def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainnet', only_grant_pk=None, what='full'): + import time + + # setup + clr_calc_start_time = timezone.now() + debug_output = [] + + # one-time data call + total_pot = float(clr_round.total_pot) + v_threshold = float(clr_round.verified_threshold) + + print(f"- starting fetch_grants at {round(time.time(),1)}") + grants = fetch_grants(clr_round, network) + + # override the grants list to the one selected + if only_grant_pk: + grants = grants.filter(pk=only_grant_pk) + + print(f"- starting get data and sum at {round(time.time(),1)}") + # collect contributions for clr_round into temp table + initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) + # open cursor and execute the groupBy sum for the round + with connection.cursor() as cursor: + counter = 0 + curr_agg = {} + # execute to populate shared state for the round + cursor.execute(initial_query) # (we could potential do better here by sharing this temp table between rounds) + for _row in cursor.fetchall(): + if not curr_agg.get(_row[0]): + curr_agg[_row[0]] = {} + curr_agg[_row[0]][_row[1]] = _row[2] + + if len(curr_agg) == 0: + print(f'- done - No Contributions for CLR {clr_round.round_num}. Exiting') + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + return + + print(f"- starting current grant calc (free of predictions) at {round(time.time(),1)}") + curr_grants_clr, _ = calculate_clr_for_donation( + None, 0, cursor, total_pot, v_threshold + ) + + if what == 'slim': + # if we are only calculating slim CLR calculations, return here and save 97% compute power + print(f"- saving slim grant calc at {round(time.time(),1)}") + total_count = len(curr_grants_clr.items()) + for grant_id, grant_calc in curr_grants_clr.items(): + counter += 1 + if counter % 10 == 0 or True: + print(f"- {counter}/{total_count} grants iter, pk:{grant_id}, at {round(time.time(),1)}") + + # update latest calcs with current distribution + grant = clr_round.grants.using('default').get(pk=grant_id) + latest_calc = grant.clr_calculations.using('default').filter(latest=True, grantclr=clr_round).order_by('-pk').first() + if not latest_calc: + print(f"- - could not find latest clr calc for {grant_id} ") + continue + clr_prediction_curve = copy.deepcopy(latest_calc.clr_prediction_curve) + clr_prediction_curve[0][1] = grant_calc['clr_amount'] if grant_calc else 0.0 # update only the existing match estimate + print(clr_prediction_curve) + clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) + grant.save() + # if we are only calculating slim CLR calculations, return here and save 97% compute power + print(f"- done calculating at {round(time.time(),1)}") + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + return + + # calculate clr given additional donations + total_count = grants.count() + + print(f"- starting grants iter at {round(time.time(),1)}") + # calculate each grant as a distinct input + for grant in grants: + # five potential additional donations plus the base case of 0 + potential_donations = [0, 1, 10, 100, 1000, 10000] + + # debug the run... + counter += 1 + if counter % 10 == 0 or True: + print(f"- {counter}/{total_count} grants iter, pk:{grant.id}, at {round(time.time(),1)}") + + # if no contributions have been made for this grant then the pairwise will fail and no match will be discovered + if not curr_agg.get(grant.id): + grants_clr = None + potential_clr = [0.0 for x in range(0, 6)] + else: + potential_clr = [] + for amount in potential_donations: + # no need to run the calculation multiple times for amount=0 (will always be the same result) + if amount == 0: + # use the current distribution calc + grants_clr = curr_grants_clr.get(grant.id) + predicted_clr = grants_clr['clr_amount'] if grants_clr else 0.0 + else: + # this is used when you want to count final distribution and ignore the prediction + if what == 'final': + # ignore the other ones + grants_clr = None + predicted_clr = 0.0 + else: + # calculate clr with each additional donation + grants_clr, predicted_clr = calculate_clr_for_donation( + grant.id, amount, cursor, total_pot, v_threshold + ) + # record each point of the predicition + potential_clr.append(predicted_clr) + + # save the result of the prediction + if save_to_db: + _grant = Grant.objects.get(pk=grant.id) + clr_prediction_curve = list(zip(potential_donations, potential_clr)) + base = clr_prediction_curve[0][1] + _grant.last_clr_calc_date = timezone.now() + _grant.next_clr_calc_date = timezone.now() + timezone.timedelta(minutes=60) + + # check that we have enough data to set the curve + can_estimate = True if base or clr_prediction_curve[1][1] or clr_prediction_curve[2][1] or clr_prediction_curve[3][1] else False + if can_estimate: + clr_prediction_curve = [[ele[0], ele[1], ele[1] - base] for ele in clr_prediction_curve ] + else: + clr_prediction_curve = [[0.0, 0.0, 0.0] for x in range(0, 6)] + print(clr_prediction_curve) + + # save the new predicition curve via the model + clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) + _grant.save() + + debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) + + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + + return debug_output diff --git a/app/grants/clr3.py b/app/grants/clr3.py new file mode 100644 index 00000000000..27d821beebe --- /dev/null +++ b/app/grants/clr3.py @@ -0,0 +1,412 @@ +# -*- coding: utf-8 -*- +"""Define the Grants application configuration. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" +import copy + +from django.db import connection +from django.utils import timezone + +import numpy as np +from grants.models import Contribution, Grant, GrantCollection +from townsquare.models import SquelchProfile + +CLR_PERCENTAGE_DISTRIBUTED = 0 + + +def fetch_grants(clr_round, network='mainnet'): + grant_filters = clr_round.grant_filters + collection_filters = clr_round.collection_filters + + grants = clr_round.grants.filter(network=network, hidden=False, active=True, is_clr_eligible=True, link_to_new_grant=None) + + if grant_filters: + # Grant Filters (grant_type, category) + grants = grants.filter(**grant_filters) + elif collection_filters: + # Collection Filters + grant_ids = GrantCollection.objects.filter(**collection_filters).values_list('grants', flat=True) + grants = grants.filter(pk__in=grant_ids) + + return grants + + +def get_summed_contribs_query(grants, created_after, created_before, multiplier, network): + # only consider contribs from current grant set + grantIds = '' + for i in range(len(grants)): + grantIds += "'" + str(grants[i].id) + "'" + (', ' if i+1 != len(grants) else '') + + summedContribs = f''' + -- group by ... sum the contributions $ value for each user + SELECT + grants.use_grant_id as grant_id, + grants_contribution.profile_for_clr_id as user_id, + SUM((grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT * {float(multiplier)}), + MAX(dashboard_profile.as_dict ->> 'trust_bonus')::FLOAT as trust_bonus + FROM grants_contribution + INNER JOIN dashboard_profile ON (grants_contribution.profile_for_clr_id = dashboard_profile.id) + INNER JOIN grants_subscription ON (grants_contribution.subscription_id = grants_subscription.id) + RIGHT JOIN ( + SELECT + grants_grant.id as grant_id, + ( + CASE + WHEN grants_grant.defer_clr_to_id IS NOT NULL THEN grants_grant.defer_clr_to_id + ELSE grants_grant.id + END + ) as use_grant_id + FROM grants_grant + ) grants ON ((grants_contribution.normalized_data ->> 'id')::FLOAT = grants.grant_id) + WHERE ( + grants_contribution.normalized_data ->> 'id' IN ({grantIds}) AND + grants_contribution.created_on >= '{created_after}' AND + grants_contribution.created_on <= '{created_before}' AND + grants_contribution.match = True AND + grants_subscription.network = '{network}' AND + grants_contribution.success = True AND + (grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT >= 0 AND + NOT ( + grants_contribution.profile_for_clr_id IN ( + SELECT squelched.profile_id FROM townsquare_squelchprofile squelched WHERE squelched.active = True + ) AND grants_contribution.profile_for_clr_id IS NOT NULL + ) + ) + GROUP BY grants.use_grant_id, grants_contribution.profile_for_clr_id; + ''' + + return summedContribs + + +def get_totals_by_pair(contrib_dict): + ''' + gets pair totals between current round, current round + + args: + aggregated contributions by pair nested dict + { + grant_id (str): { + user_id (str): aggregated_amount (float) + } + } + + returns: + pair totals between current round + {user_id (str): {user_id (str): pair_total (float)}} + + ''' + tot_overlap = {} + + # start pairwise match + for _, contribz in contrib_dict.items(): + for k1, v1 in contribz.items(): + if k1 not in tot_overlap: + tot_overlap[k1] = {} + + # pairwise matches to current round + for k2, v2 in contribz.items(): + if k2 not in tot_overlap[k1]: + tot_overlap[k1][k2] = 0 + tot_overlap[k1][k2] += (v1 * v2) ** 0.5 + + return tot_overlap + + +def calculate_clr(aggregated_contributions, pair_totals, trust_dict, v_threshold, total_pot): + ''' + calculates the clr amount at the given threshold and total pot + args: + aggregated contributions by pair nested dict + { + grant_id (str): { + user_id (str): aggregated_amount (float) + } + } + pair_totals + {user_id (str): {user_id (str): pair_total (float)}} + trust_dict + {user_id (str): trust_score (float)} + v_threshold + float + total_pot + float + + returns: + total clr award by grant, analytics, normalized by the normalization factor + [{'id': proj, 'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}] + saturation point + boolean + ''' + + bigtot = 0 + totals = {} + + for proj, contribz in aggregated_contributions.items(): + tot = 0 + _num = 0 + _sum = 0 + + # start pairwise matches + for k1, v1 in contribz.items(): + _num += 1 + _sum += v1 + + # pairwise matches to current round + for k2, v2 in contribz.items(): + if int(k2) > int(k1): + tot += ((v1 * v2) ** 0.5) / (pair_totals[k1][k2] / (v_threshold * max(trust_dict[k2], trust_dict[k1])) + 1) + + if type(tot) == complex: + tot = float(tot.real) + + bigtot += tot + totals[proj] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot} + + global CLR_PERCENTAGE_DISTRIBUTED + + if bigtot >= total_pot: # saturation reached + # print(f'saturation reached. Total Pot: ${total_pot} | Total Allocated ${bigtot}. Normalizing') + CLR_PERCENTAGE_DISTRIBUTED = 100 + for key, t in totals.items(): + t['clr_amount'] = ((t['clr_amount'] / bigtot) * total_pot) + else: + CLR_PERCENTAGE_DISTRIBUTED = (bigtot / total_pot) * 100 + if bigtot == 0: + bigtot = 1 + percentage_increase = np.log(total_pot / bigtot) / 100 + for key, t in totals.items(): + t['clr_amount'] = t['clr_amount'] * (1 + percentage_increase) + return totals + + +def run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot): + ''' + clubbed function that runs all calculation functions + + args: + curr_agg : + { + grantId (int): { + profileId (str): amount (float) + } + } + trust_dict : + { + profileId (str): trust_bonus (float) + } + v_threshold : float + total_pot : float + + returns: + grants clr award amounts (dict) + ''' + + # get pair totals + ptots = get_totals_by_pair(curr_agg) + + # clr calcluation + totals = calculate_clr(curr_agg, ptots, trust_dict, v_threshold, total_pot) + + return totals + + +def calculate_clr_for_donation(curr_agg, trust_dict, grant_id, amount, v_threshold, total_pot): + ''' + clubbed function that runs all calculation functions and returns the result for a single grant_id + + args: + curr_agg : + { + grantId (int): { + profileId (str): amount (float) + } + } + trust_dict : + { + profileId (str): trust_bonus (float) + } + grant_id ; int + amount ; int + v_threshold : float + total_pot : float + + returns: + (grant clr award amounts (dict), clr_amount (float), number_contributions (int), contribution_amount (float)) + ''' + + # make sure contributions exist + if curr_agg.get(grant_id) or not grant_id: + # find grant in contributions list and add donation + if amount: + trust_dict['999999999999'] = 1 + curr_agg[grant_id]['999999999999'] = amount + + grants_clr = run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot) + + # find grant we added the contribution to and get the new clr amount + if grants_clr.get(grant_id): + grant_clr = grants_clr.get(grant_id) + return ( + grant_clr, + grant_clr['clr_amount'], + grant_clr['number_contributions'], + grant_clr['contribution_amount'] + ) + else: + grants_clr = None + + # print(f'info: no contributions found for grant {grant}') + return (grants_clr, 0.0, 0, 0.0) + + +def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainnet', only_grant_pk=None, what='full'): + import time + + # setup + clr_calc_start_time = timezone.now() + debug_output = [] + + # one-time data call + total_pot = float(clr_round.total_pot) + v_threshold = float(clr_round.verified_threshold) + + print(f"- starting fetch_grants at {round(time.time(),1)}") + # grants, contributions = fetch_data(clr_round, network) + grants = fetch_grants(clr_round, network) + + if only_grant_pk: + grants = grants.filter(pk=only_grant_pk) + + print(f"- starting get data and sum at {round(time.time(),1)}") + # collect contributions for clr_round into temp table + initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) + # open cursor and execute the groupBy sum for the round + with connection.cursor() as cursor: + counter = 0 + curr_agg = {} + trust_dict = {} + # execute to populate shared state for the round + cursor.execute(initial_query) # (we could potential do better here by sharing this temp table between rounds) + for _row in cursor.fetchall(): + if not curr_agg.get(_row[0]): + curr_agg[_row[0]] = {} + trust_dict[_row[1]] = _row[3] + curr_agg[_row[0]][_row[1]] = _row[2] + + if len(curr_agg) == 0: + print(f'- done - no Contributions for CLR {clr_round.round_num}. Exiting') + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + return + + print(f"- starting current grant calc (free of predictions) at {round(time.time(),1)}") + curr_grants_clr = run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot) + + if what == 'slim': + # if we are only calculating slim CLR calculations, return here and save 97% compute power + print(f"- saving slim grant calc at {round(time.time(),1)}") + total_count = len(curr_grants_clr.items()) + for pk, grant_calc in curr_grants_clr.items(): + counter += 1 + if counter % 10 == 0 or True: + print(f"- {counter}/{total_count} grants iter, pk:{pk}, at {round(time.time(),1)}") + + # update latest calcs with current distribution + grant_calc = curr_grants_clr[pk] + grant = clr_round.grants.using('default').get(pk=pk) + latest_calc = grant.clr_calculations.using('default').filter(latest=True, grantclr=clr_round).order_by('-pk').first() + if not latest_calc: + print(f"- - could not find latest clr calc for {grant.pk} ") + continue + clr_prediction_curve = copy.deepcopy(latest_calc.clr_prediction_curve) + clr_prediction_curve[0][1] = grant_calc['clr_amount'] # update only the existing match estimate + print(clr_prediction_curve) + clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) + grant.save() + # if we are only calculating slim CLR calculations, return here and save 97% compute power + print(f"- done calculating at {round(time.time(),1)}") + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + return + + print(f"- starting grants iter at {round(time.time(),1)}") + # calculate clr given additional donations + total_count = grants.count() + for grant in grants: + # five potential additional donations plus the base case of 0 + potential_donations = [0, 1, 10, 100, 1000, 10000] + + # debug the run... + counter += 1 + if counter % 10 == 0 or True: + print(f"- {counter}/{total_count} grants iter, pk:{grant.id}, at {round(time.time(),1)}") + + # if no contributions have been made for this grant then the pairwise will fail and no match will be discovered + if not curr_agg.get(grant.id): + grants_clr = None + potential_clr = [0.0 for x in range(0, 6)] + else: + potential_clr = [] + for amount in potential_donations: + # no need to run the calculation multiple times for amount=0 (will always be the same result) + if amount == 0: + # use the current distribution calc + grants_clr = curr_grants_clr.get(grant.id) + predicted_clr = grants_clr['clr_amount'] if grants_clr else 0.0 + else: + # this is used when you want to count final distribution and ignore the prediction + if what == 'final': + # ignore the other ones + grants_clr = None + predicted_clr = 0.0 + else: + # calculate clr with each additional donation + grants_clr, predicted_clr, _, _ = calculate_clr_for_donation( + curr_agg, trust_dict, grant.id, amount, v_threshold, total_pot + ) + + # reset potential_donations + if amount and curr_agg.get(grant.id): + del curr_agg[grant.id]['999999999999'] + + # record each point of the predicition + potential_clr.append(predicted_clr) + + # save the result of the prediction + if save_to_db: + _grant = Grant.objects.get(pk=grant.id) + clr_prediction_curve = list(zip(potential_donations, potential_clr)) + base = clr_prediction_curve[0][1] + _grant.last_clr_calc_date = timezone.now() + _grant.next_clr_calc_date = timezone.now() + timezone.timedelta(minutes=60) + + # check that we have enough data to set the curve + can_estimate = True if base or clr_prediction_curve[1][1] or clr_prediction_curve[2][1] or clr_prediction_curve[3][1] else False + if can_estimate: + clr_prediction_curve = [[ele[0], ele[1], ele[1] - base] for ele in clr_prediction_curve ] + else: + clr_prediction_curve = [[0.0, 0.0, 0.0] for x in range(0, 6)] + print(clr_prediction_curve) + + # save the new predicition curve via the model + clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) + _grant.save() + + debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) + + print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") + + return debug_output diff --git a/app/grants/management/commands/analytics_clr2.py b/app/grants/management/commands/analytics_clr2.py new file mode 100644 index 00000000000..fa0f6f1bada --- /dev/null +++ b/app/grants/management/commands/analytics_clr2.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +"""Define the Grant subminer management command. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" + +from django.core.management.base import BaseCommand +from django.db import connection +from django.utils import timezone + +from dashboard.models import Profile +from grants.clr2 import calculate_clr_for_donation, fetch_grants, get_summed_contribs_query +from grants.models import GrantCLR + + +def analytics_clr(from_date=None, clr_round=None, network='mainnet'): + # setup + # clr_calc_start_time = timezone.now() + debug_output = [['grant_id', 'grant_title', 'number_contributions', 'contribution_amount', 'clr_amount']] + + # one-time data call + total_pot = float(clr_round.total_pot) + v_threshold = float(clr_round.verified_threshold) + + print(total_pot) + + grants = fetch_grants(clr_round, network) + # collect contributions for clr_round into temp table + initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) + # open cursor and execute the groupBy sum for the round + with connection.cursor() as cursor: + # execute to populate shared state for the round + cursor.execute(initial_query) + # calculate clr analytics output + for grant in grants: + _, clr_amount = calculate_clr_for_donation( + grant.id, + 0, + cursor, + total_pot, + v_threshold + ) + debug_output.append([grant.id, grant.title, grant.positive_round_contributor_count, float(grant.amount_received_in_round), clr_amount]) + + return debug_output + + + +class Command(BaseCommand): + + help = 'calculate clr base analytic results for all clr rounds or for a specific clr round' + + def add_arguments(self, parser): + parser.add_argument('network', type=str, default='mainnet', choices=['rinkeby', 'mainnet']) + parser.add_argument('clr_pk', type=str, default="all") + + + def handle(self, *args, **options): + + network = options['network'] + clr_pk = options['clr_pk'] + + if clr_pk == "all": + active_clr_rounds = GrantCLR.objects.filter(is_active=True) + else: + active_clr_rounds = GrantCLR.objects.filter(pk=clr_pk) + + if active_clr_rounds: + for clr_round in active_clr_rounds: + print(f"calculating CLR results for round: {clr_round.round_num} {clr_round.sub_round_slug}") + analytics = analytics_clr( + from_date=timezone.now(), + clr_round=clr_round, + network=network + ) + print(analytics) + print(f"finished CLR results for round: {clr_round.round_num} {clr_round.sub_round_slug}") + + else: + print("No active CLRs found") diff --git a/app/grants/management/commands/analytics_clr3.py b/app/grants/management/commands/analytics_clr3.py new file mode 100644 index 00000000000..6bc715e633e --- /dev/null +++ b/app/grants/management/commands/analytics_clr3.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""Define the Grant subminer management command. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" + +from django.core.management.base import BaseCommand +from django.db import connection +from django.utils import timezone + +from grants.clr3 import calculate_clr_for_donation, fetch_grants, get_summed_contribs_query +from grants.models import GrantCLR + + +def analytics_clr(from_date=None, clr_round=None, network='mainnet'): + # setup + # clr_calc_start_time = timezone.now() + debug_output = [['grant_id', 'grant_title', 'number_contributions', 'contribution_amount', 'clr_amount']] + + # one-time data call + total_pot = float(clr_round.total_pot) + v_threshold = float(clr_round.verified_threshold) + + print(total_pot) + + grants = fetch_grants(clr_round, network) + + # collect contributions for clr_round into temp table + initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) + + # open cursor and execute the groupBy sum for the round + with connection.cursor() as cursor: + curr_agg = {} + trust_dict = {} + # execute to populate shared state for the round + cursor.execute(initial_query) # (we could potential do better here by sharing this temp table between rounds) + for _row in cursor.fetchall(): + if not curr_agg.get(_row[0]): + curr_agg[_row[0]] = {} + + trust_dict[_row[1]] = _row[3] + curr_agg[_row[0]][_row[1]] = _row[2] + + # calculate clr analytics output + for grant in grants: + _, clr_amount, num_contribs, contrib_amount = calculate_clr_for_donation( + curr_agg, trust_dict, grant.id, 0, v_threshold, total_pot + ) + # debug_output.append([grant.id, grant.title, num_contribs, contrib_amount, clr_amount]) + debug_output.append([grant.id, grant.title, grant.positive_round_contributor_count, float(grant.amount_received_in_round), clr_amount]) + + return debug_output + + + +class Command(BaseCommand): + + help = 'calculate clr base analytic results for all clr rounds or for a specific clr round' + + def add_arguments(self, parser): + parser.add_argument('network', type=str, default='mainnet', choices=['rinkeby', 'mainnet']) + parser.add_argument('clr_pk', type=str, default="all") + + + def handle(self, *args, **options): + + network = options['network'] + clr_pk = options['clr_pk'] + + if clr_pk == "all": + active_clr_rounds = GrantCLR.objects.filter(is_active=True) + else: + active_clr_rounds = GrantCLR.objects.filter(pk=clr_pk) + + if active_clr_rounds: + for clr_round in active_clr_rounds: + print(f"calculating CLR results for round: {clr_round.round_num} {clr_round.sub_round_slug}") + analytics = analytics_clr( + from_date=timezone.now(), + clr_round=clr_round, + network=network + ) + print(analytics) + print(f"finished CLR results for round: {clr_round.round_num} {clr_round.sub_round_slug}") + + else: + print("No active CLRs found") diff --git a/app/grants/management/commands/estimate_clr.py b/app/grants/management/commands/estimate_clr.py index a425e1b2633..33cec050cae 100644 --- a/app/grants/management/commands/estimate_clr.py +++ b/app/grants/management/commands/estimate_clr.py @@ -45,7 +45,7 @@ def handle(self, *args, **options): clr_pk = options['clr_pk'] what = options['what'] sync = options['sync'] - print (network, clr_pk, what, sync) + print ('clr', network, clr_pk, what, sync) if clr_pk and clr_pk.isdigit(): active_clr_rounds = GrantCLR.objects.filter(pk=clr_pk) diff --git a/app/grants/management/commands/estimate_clr2.py b/app/grants/management/commands/estimate_clr2.py new file mode 100644 index 00000000000..ac395a9471a --- /dev/null +++ b/app/grants/management/commands/estimate_clr2.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +"""Define the Grant subminer management command. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from grants.clr2 import predict_clr +from grants.models import GrantCLR +from grants.tasks import process_predict_clr + + +class Command(BaseCommand): + + help = 'calculate CLR estimates for all grants' + + def add_arguments(self, parser): + parser.add_argument('network', type=str, default='mainnet', choices=['rinkeby', 'mainnet']) + parser.add_argument('clr_pk', type=str, default="all") + parser.add_argument('what', type=str, default="full") + parser.add_argument('sync', type=str, default="false") + # slim = just run 0 contribution match upcate calcs + # full, run [0, 1, 10, 100, calcs across all grants] + + + def handle(self, *args, **options): + + network = options['network'] + clr_pk = options['clr_pk'] + what = options['what'] + sync = options['sync'] + print ('clr2', network, clr_pk, what, sync) + + if clr_pk and clr_pk.isdigit(): + active_clr_rounds = GrantCLR.objects.filter(pk=clr_pk) + else: + active_clr_rounds = GrantCLR.objects.filter(is_active=True) + + if active_clr_rounds: + for clr_round in active_clr_rounds: + if sync == 'true': + # run it sync -> useful for payout / debugging + predict_clr( + save_to_db=True, + from_date=timezone.now(), + clr_round=clr_round, + network=network, + what=what, + ) + else: + # runs it as celery task. + process_predict_clr( + save_to_db=True, + from_date=timezone.now(), + clr_round=clr_round, + network=network, + what=what, + ) + else: + print("No active CLRs found") diff --git a/app/grants/management/commands/estimate_clr3.py b/app/grants/management/commands/estimate_clr3.py new file mode 100644 index 00000000000..8941d53ae30 --- /dev/null +++ b/app/grants/management/commands/estimate_clr3.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +"""Define the Grant subminer management command. + +Copyright (C) 2021 Gitcoin Core + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +""" + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from grants.clr3 import predict_clr +from grants.models import GrantCLR +from grants.tasks import process_predict_clr + + +class Command(BaseCommand): + + help = 'calculate CLR estimates for all grants' + + def add_arguments(self, parser): + parser.add_argument('network', type=str, default='mainnet', choices=['rinkeby', 'mainnet']) + parser.add_argument('clr_pk', type=str, default="all") + parser.add_argument('what', type=str, default="full") + parser.add_argument('sync', type=str, default="false") + # slim = just run 0 contribution match upcate calcs + # full, run [0, 1, 10, 100, calcs across all grants] + + + def handle(self, *args, **options): + + network = options['network'] + clr_pk = options['clr_pk'] + what = options['what'] + sync = options['sync'] + print ('clr3', network, clr_pk, what, sync) + + if clr_pk and clr_pk.isdigit(): + active_clr_rounds = GrantCLR.objects.filter(pk=clr_pk) + else: + active_clr_rounds = GrantCLR.objects.filter(is_active=True) + + if active_clr_rounds: + for clr_round in active_clr_rounds: + if sync == 'true': + # run it sync -> useful for payout / debugging + predict_clr( + save_to_db=True, + from_date=timezone.now(), + clr_round=clr_round, + network=network, + what=what, + ) + else: + # runs it as celery task. + process_predict_clr( + save_to_db=True, + from_date=timezone.now(), + clr_round=clr_round, + network=network, + what=what, + ) + else: + print("No active CLRs found")