CLR integration branch (#9736)

* GITC-475: Adds active to GrantCLRCalculation to mark ongoing rounds * GITC-475: Command to set latest on historic calc data * GITC-475: Uses active in place of latest to retain hold over most recent calcs * GITC-475: resolves multiple leaf nodes in migrations * fix: fixes isort/linting issues * GITC-500: Adds ability to recalculate inactive rounds * GITC-475: Use only active calcs to set Grants clr_prediction_curve * GITC-500: Prep form merge - we should make this check on active once GITC-475 is merged * GITC-495: flattens queried columns and adds indexing * add grant_clr_percentage_cap * fix: combines migrations * fix: moves is_active check to record_clr_prediction_curve -> active * fix: typos * fix: defaults percentage_cap to 100% if absent * fix: corrects migration dependency * fix: conflicting migrations Co-authored-by: Aditya Anand M C <[email protected]>
gitcoinco · Nov 26, 2021 · 4f3d0de · 4f3d0de
1 parent 0c6a92d
commit 4f3d0de
Show file tree

Hide file tree

Showing 21 changed files with 288 additions and 101 deletions.
diff --git a/app/dashboard/management/commands/update_trust_bonus.py b/app/dashboard/management/commands/update_trust_bonus.py
@@ -38,9 +38,6 @@ def handle(self, *args, **options):
         print(profiles.count())
         for profile in profiles.iterator():
             if (options['call_now']):
-                params = profile.as_dict
-                params['trust_bonus'] = profile.trust_bonus
-                print("Saving - %s - %s" % (profile.handle, params['trust_bonus']))
-                profile.save()
+                update_trust_bonus(profile.pk)
             else:
                 update_trust_bonus.delay(profile.pk)
diff --git a/app/dashboard/migrations/0196_profile_trust_bonus.py b/app/dashboard/migrations/0196_profile_trust_bonus.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-11-16 03:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0195_auto_20211124_0639'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='profile',
+            name='trust_bonus',
+            field=models.DecimalField(decimal_places=2, default=0.5, help_text='Trust Bonus score based on verified services', max_digits=5),
+        ),
+    ]
diff --git a/app/dashboard/models.py b/app/dashboard/models.py
@@ -3014,6 +3014,9 @@ class Profile(SuperModel):
     idena_address = models.CharField(max_length=128, null=True, unique=True, blank=True)
     idena_status = models.CharField(max_length=32, null=True, blank=True)
 
+    # store the trust bonus on the model itself
+    trust_bonus = models.DecimalField(default=0.5, decimal_places=2, max_digits=5, help_text='Trust Bonus score based on verified services')
+
     def update_idena_status(self):
         self.idena_status = get_idena_status(self.idena_address)
 
@@ -3026,41 +3029,6 @@ def update_idena_status(self):
     def shadowbanned(self):
         return self.squelches.filter(active=True).exists()
 
-    @property
-    def trust_bonus(self):
-        # returns a percentage trust bonus, for this curent user.
-        # trust bonus starts at 50% and compounds for every new verification added
-        # to a max of 150%
-        tb = 0.5
-        if self.is_poh_verified:
-            tb += 0.50
-        if self.is_brightid_verified:
-            tb += 0.50
-        if self.is_idena_verified:
-            tb += 0.50
-        if self.is_poap_verified:
-            tb += 0.25
-        if self.is_ens_verified:
-            tb += 0.25
-        if self.sms_verification:
-            tb += 0.15
-        if self.is_google_verified:
-            tb += 0.15
-        if self.is_twitter_verified:
-            tb += 0.15
-        if self.is_facebook_verified:
-            tb += 0.15
-        # if self.is_duniter_verified:
-        #     tb *= 1.001
-        qd_tb = 0
-        for player in self.players.all():
-            new_score = 0
-            if player.tokens_in:
-                new_score = min(player.tokens_in / 100, 0.20)
-            qd_tb = max(qd_tb, new_score)
-        return min(1.5, tb)
-
-
     @property
     def is_blocked(self):
         if not self.user:
@@ -4389,7 +4357,6 @@ def to_dict(self):
             'card_title': f'@{self.handle} | Gitcoin',
             'org_works_with': org_works_with,
             'card_desc': desc,
-            'trust_bonus': self.trust_bonus,
             'avatar_url': self.avatar_url_with_gitcoin_logo,
             'count_bounties_completed': total_fulfilled,
             'works_with_collected': works_with_collected,

diff --git a/app/dashboard/tasks.py b/app/dashboard/tasks.py
@@ -255,11 +255,46 @@ def update_trust_bonus(self, pk):
     :param pk:
     :return:
     """
+    # returns a percentage trust bonus, for this current user.
+    # trust bonus starts at 50% and compounds for every new verification added
+    # to a max of 150%
+    tb = 0.5
     profile = Profile.objects.get(pk=pk)
-    params = profile.as_dict
-    if profile.trust_bonus != params.get('trust_bonus', None):
-        params['trust_bonus'] = profile.trust_bonus
-        print("Saving - %s - %s" % (profile.handle, params['trust_bonus']))
+    if profile.is_poh_verified:
+        tb += 0.50
+    if profile.is_brightid_verified:
+        tb += 0.50
+    if profile.is_idena_verified:
+        tb += 0.50
+    if profile.is_poap_verified:
+        tb += 0.25
+    if profile.is_ens_verified:
+        tb += 0.25
+    if profile.sms_verification:
+        tb += 0.15
+    if profile.is_google_verified:
+        tb += 0.15
+    if profile.is_twitter_verified:
+        tb += 0.15
+    if profile.is_facebook_verified:
+        tb += 0.15
+    # if profile.is_duniter_verified:
+    #     tb *= 1.001
+    qd_tb = 0
+    for player in profile.players.all():
+        new_score = 0
+        if player.tokens_in:
+            new_score = min(player.tokens_in / 100, 0.20)
+        qd_tb = max(qd_tb, new_score)
+
+    # cap the trust_bonus score at 1.5
+    tb = min(1.5, tb)
+
+    print("Saving - %s - %s - %s" % (profile.handle, profile.trust_bonus, tb))
+
+    # save the new score
+    if profile.trust_bonus != tb:
+        profile.trust_bonus = tb
         profile.save()
 
 

diff --git a/app/grants/admin.py b/app/grants/admin.py
@@ -202,7 +202,7 @@ def response_change(self, request, obj):
         if "_calc_clr" in request.POST:
             from grants.tasks import recalc_clr
             recalc_clr.delay(obj.pk)
-            self.message_user(request, "recaclulation of clr queued")
+            self.message_user(request, "recalculation of clr queued")
         if "_request_more_info" in request.POST:
             more_info = request.POST.get('more_info')
             grant_more_info_required(obj, more_info)
@@ -489,26 +489,30 @@ def stats_link(self, instance):
     def response_change(self, request, obj):
         if "_recalculate_clr" in request.POST:
             from grants.tasks import recalc_clr
-            for grant in obj.grants:
-                recalc_clr.delay(grant.pk)
-            self.message_user(request, "submitted recaclulation to queue")
+            selected_clr = request.POST.get('_selected_clr', False)
+            if selected_clr:
+                recalc_clr.delay(False, int(selected_clr))
+                self.message_user(request, f"submitted recalculation of GrantCLR:{ selected_clr } to queue")
+            else:
+                recalc_clr.delay(False)
+                self.message_user(request, "submitted recalculation to queue")
 
         if "_set_current_grant_clr_calculations_to_false" in request.POST:
-            latest_calculations = GrantCLRCalculation.objects.filter(grantclr=obj, latest=True)
+            active_calculations = GrantCLRCalculation.objects.filter(grantclr=obj, active=True)
 
-            if latest_calculations.count() == 0:
-                self.message_user(request, "Latest Flag is already false. No action taken")
+            if active_calculations.count() == 0:
+                self.message_user(request, "Active Flag is already false. No action taken")
             else:
-                latest_calculations.update(latest=False)
-                self.message_user(request, "Current Grant CLR Calculations's latest flag is set to false")
+                active_calculations.update(active=False)
+                self.message_user(request, "Current Grant CLR Calculations's active flag is set to false")
 
         if "_set_all_grant_clr_calculations_to_false" in request.POST:
-            latest_calculations = GrantCLRCalculation.objects.filter(latest=True)
-            if latest_calculations.count() == 0:
-                self.message_user(request, "Latest Flag is already false for all CLRs. No action taken")
+            active_calculations = GrantCLRCalculation.objects.filter(active=True)
+            if active_calculations.count() == 0:
+                self.message_user(request, "Active Flag is already false for all CLRs. No action taken")
             else:
-                latest_calculations.update(latest=False)
-                self.message_user(request, "All Grant CLR Calculations's latest flag is set to false")
+                active_calculations.update(active=False)
+                self.message_user(request, "All Grant CLR Calculations's active flag is set to false")
 
         return redirect(obj.admin_url)
 

diff --git a/app/grants/clr.py b/app/grants/clr.py
@@ -23,7 +23,7 @@
 from django.utils import timezone
 
 import numpy as np
-from grants.clr_data_src import fetch_contributions, fetch_grants
+from grants.clr_data_src import fetch_contributions, fetch_grants, fetch_summed_contributions
 
 
 def populate_data_for_clr(grants, contributions, clr_round):
@@ -58,7 +58,7 @@ def populate_data_for_clr(grants, contributions, clr_round):
     _contributions = list(contributions.filter(created_on__gte=clr_start_date, created_on__lte=clr_end_date).prefetch_related('profile_for_clr', 'subscription'))
     _contributions_by_id = {}
     for ele in _contributions:
-        key = ele.normalized_data.get('id')
+        key = ele.grant_id
         if key not in _contributions_by_id.keys():
             _contributions_by_id[key] = []
         _contributions_by_id[key].append(ele)
@@ -89,7 +89,7 @@ def populate_data_for_clr(grants, contributions, clr_round):
         # contributions
         if len(contributing_profile_ids) > 0:
             for profile_id, trust_bonus in contributing_profile_ids:
-                sum_of_each_profiles_contributions = sum(ele.normalized_data.get('amount_per_period_usdt') for ele in contributions_by_id[profile_id]) * float(clr_round.contribution_multiplier)
+                sum_of_each_profiles_contributions = sum(ele.amount_per_period_usdt for ele in contributions_by_id[profile_id]) * clr_round.contribution_multiplier
 
                 summed_contributions.append({
                     'id': str(profile_id),
@@ -113,7 +113,7 @@ def translate_data(grants_data):
             django grant data structure
                 {
                     'id': (string) ,
-                    'contibutions' : [
+                    'contributions' : [
                         {
                             contributor_profile (str) : summed_contributions
                         }
@@ -195,12 +195,12 @@ def get_totals_by_pair(contrib_dict):
             for k2, v2 in contribz.items():
                 if k2 not in pair_totals[k1]:
                     pair_totals[k1][k2] = 0
-                pair_totals[k1][k2] += (v1 * v2) ** 0.5
+                pair_totals[k1][k2] += float(v1 * v2) ** 0.5
 
     return pair_totals
 
 
-def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot):
+def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot, grant_clr_percentage_cap):
     '''
         calculates the clr amount at the given threshold and total pot
         args:
@@ -226,6 +226,7 @@ def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot):
     '''
     bigtot = 0
     totals = {}
+    match_cap_per_grant = total_pot * (grant_clr_percentage_cap / 100)
 
     for proj, contribz in curr_agg.items():
         tot = 0
@@ -240,11 +241,15 @@ def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot):
             # pairwise matches to current round
             for k2, v2 in contribz.items():
                 if int(k2) > int(k1):
-                    tot += ((v1 * v2) ** 0.5) / (pair_totals[k1][k2] / (v_threshold * max(trust_dict[k2], trust_dict[k1])) + 1)
+                    tot += (float(v1 * v2) ** 0.5) / (pair_totals[k1][k2] / (v_threshold * float(max(trust_dict[k2], trust_dict[k1]))) + 1)
 
         if type(tot) == complex:
             tot = float(tot.real)
 
+        # ensure CLR match for a grant in CLR round does not exceed 2.5 of the total pot
+        if grant_clr_percentage_cap and tot > match_cap_per_grant:
+            tot = match_cap_per_grant
+
         bigtot += tot
         totals[proj] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}
 
@@ -309,8 +314,8 @@ def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_thresho
             # which will only be paired with other contributions for this grant - because of this we can skip rebuilding the pair_total
             # and only have to consider the curr grants contributions and the prediction amount - this saves a huge amount of compute O(n)
             for k2, v2 in contribz.items():
-                pt = ((amount * v2) ** 0.5)
-                tot +=  pt / (pt / (v_threshold * max(trust_dict[k2], 1)) + 1)
+                pt = (float(amount * v2) ** 0.5)
+                tot +=  pt / (pt / (v_threshold * float(max(trust_dict[k2], 1))) + 1)
 
             if type(tot) == complex:
                 tot = float(tot.real)
@@ -362,7 +367,7 @@ def normalise(bigtot, totals, total_pot):
     return totals
 
 
-def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainnet', only_grant_pk=None, what='full'):
+def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainnet', only_grant_pk=None, what='full', use_sql=False):
     # setup
     counter = 0
     debug_output = []
@@ -375,15 +380,20 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
     print(f"- starting fetch_grants at {round(time.time(),1)}")
     grants = fetch_grants(clr_round, network)
 
-    print(f"- starting fetch_contributions at {round(time.time(),1)}")
-    contributions = fetch_contributions(clr_round, network)
+    # collect data using sql or django (to group+sum)
+    if use_sql:
+        print(f"- starting get data and sum at {round(time.time(),1)}")
+        curr_agg, trust_dict = fetch_summed_contributions(grants, clr_round, network)
+    else:
+        print(f"- starting fetch_contributions at {round(time.time(),1)}")
+        contributions = fetch_contributions(clr_round, network)
 
-    print(f"- starting sum (of {contributions.count()} contributions) at {round(time.time(),1)}")
-    grant_contributions_curr = populate_data_for_clr(grants, contributions, clr_round)
-    curr_round, trust_dict = translate_data(grant_contributions_curr)
+        print(f"- starting sum (of {contributions.count()} contributions) at {round(time.time(),1)}")
+        grant_contributions_curr = populate_data_for_clr(grants, contributions, clr_round)
+        curr_round, trust_dict = translate_data(grant_contributions_curr)
 
-    # this aggregates the data into the expected format
-    curr_agg = aggregate_contributions(curr_round)
+        # this aggregates the data into the expected format
+        curr_agg = aggregate_contributions(curr_round)
 
     if len(curr_agg) == 0:
         print(f'- done - no Contributions for CLR {clr_round.round_num}. Exiting')
@@ -393,7 +403,9 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
     print(f"- starting current distributions calc at {round(time.time(),1)}")
     # aggregate pairs and run calculation to get current distribution
     pair_totals = get_totals_by_pair(curr_agg)
-    bigtot, totals = calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot)
+
+    grant_clr_percentage_cap = clr_round.grant_clr_percentage_cap if clr_round.grant_clr_percentage_cap else 100
+    bigtot, totals = calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot, grant_clr_percentage_cap)
 
     # normalise against a deepcopy of the totals to avoid mutations
     curr_grants_clr = normalise(bigtot, copy.deepcopy(totals), total_pot)
@@ -464,7 +476,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
                             bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant.id, amount
                         )
 
-                # record each point of the predicition
+                # record each point of the prediction
                 potential_clr.append(predicted_clr)
 
         # save the result of the prediction

diff --git a/app/grants/clr_data_src.py b/app/grants/clr_data_src.py
@@ -102,8 +102,8 @@ def fetch_summed_contributions(grants, clr_round, network='mainnet'):
         SELECT
             grants.use_grant_id as grant_id,
             grants_contribution.profile_for_clr_id as user_id,
-            SUM((grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT * {float(multiplier)}),
-            MAX(dashboard_profile.as_dict ->> 'trust_bonus')::FLOAT as trust_bonus
+            SUM(grants_contribution.amount_per_period_usdt * {float(multiplier)}),
+            MAX(dashboard_profile.trust_bonus)::FLOAT as trust_bonus
         FROM grants_contribution
         INNER JOIN dashboard_profile ON (grants_contribution.profile_for_clr_id = dashboard_profile.id)
         INNER JOIN grants_subscription ON (grants_contribution.subscription_id = grants_subscription.id)
@@ -117,15 +117,15 @@ def fetch_summed_contributions(grants, clr_round, network='mainnet'):
                     END
                 ) as use_grant_id
             FROM grants_grant
-        ) grants ON ((grants_contribution.normalized_data ->> 'id')::FLOAT = grants.grant_id)
+        ) grants ON (grants_contribution.grant_id = grants.grant_id)
         WHERE (
-            grants_contribution.normalized_data ->> 'id' IN ({grantIds}) AND
+            grants_contribution.grant_id IN ({grantIds}) AND
             grants_contribution.created_on >= '{clr_start_date}' AND
             grants_contribution.created_on <= '{clr_end_date}' AND
             grants_contribution.match = True AND
             grants_subscription.network = '{network}' AND
             grants_contribution.success = True AND
-            (grants_contribution.normalized_data ->> 'amount_per_period_usdt')::FLOAT >= 0 AND
+            grants_contribution.amount_per_period_usdt >= 0 AND
             NOT (
                 grants_contribution.profile_for_clr_id IN (
                     SELECT squelched.profile_id FROM townsquare_squelchprofile squelched WHERE squelched.active = True