From 1a3ad3484e0e038ad73a99b256ab6087b11a0436 Mon Sep 17 00:00:00 2001 From: Andrew Redden Date: Fri, 4 Sep 2020 11:42:41 -0300 Subject: [PATCH] =?UTF-8?q?updated=20user=20directory=20models/indexers=20?= =?UTF-8?q?to=20reflect=20proper=20production=20d=E2=80=A6=20(#7307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated user directory models/indexers to reflect proper production data set types * change haystack to use a new env variable to talk to its own cluster * remove the interpolation of the variable * update proxyview url --- app/app/settings.py | 4 +- .../migrations/0143_userdirectory.py | 43 ++++++++++--------- app/dashboard/models.py | 41 +++++++++--------- app/dashboard/search_indexes.py | 19 ++++---- app/dashboard/views.py | 2 +- docker-compose.yml | 2 +- 6 files changed, 58 insertions(+), 53 deletions(-) diff --git a/app/app/settings.py b/app/app/settings.py index 2ac37cd061f..b9756a7f474 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -824,10 +824,12 @@ def callback(request): ELASTIC_SEARCH_URL = env('ELASTIC_SEARCH_URL', default='') +HAYSTACK_ELASTIC_SEARCH_URL = env('HAYSTACK_ELASTIC_SEARCH_URL', default='') + HAYSTACK_CONNECTIONS = { 'default': { 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': f"{ELASTIC_SEARCH_URL}:9200", + 'URL': HAYSTACK_ELASTIC_SEARCH_URL, 'INDEX_NAME': 'haystack', }, } diff --git a/app/dashboard/migrations/0143_userdirectory.py b/app/dashboard/migrations/0143_userdirectory.py index 0a366be0710..05acc98d33d 100644 --- a/app/dashboard/migrations/0143_userdirectory.py +++ b/app/dashboard/migrations/0143_userdirectory.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.4 on 2020-08-31 13:50 +# Generated by Django 2.2.4 on 2020-08-31 20:00 import django.contrib.postgres.fields from django.db import migrations, models @@ -15,21 +15,22 @@ class Migration(migrations.Migration): name='UserDirectory', fields=[ ('profile_id', models.CharField(max_length=255, primary_key=True, serialize=False)), - ('join_date', models.CharField(max_length=255)), - ('github_created_at', models.CharField(max_length=255)), - ('first_name', models.CharField(max_length=255)), - ('last_name', models.CharField(max_length=255)), + ('join_date', models.CharField(max_length=255, null=True)), + ('github_created_at', models.CharField(max_length=255, null=True)), + ('first_name', models.CharField(max_length=255, null=True)), + ('last_name', models.CharField(max_length=255, null=True)), ('email', models.EmailField(max_length=254)), - ('handle', models.CharField(max_length=255)), + ('handle', models.CharField(max_length=255, null=True)), ('sms_verification', models.BooleanField()), - ('persona', models.CharField(max_length=255)), + ('persona', models.CharField(max_length=255, null=True)), ('rank_coder', models.IntegerField()), + ('rank_funder', models.IntegerField()), ('num_hacks_joined', models.IntegerField()), - ('which_hacks_joined', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None)), + ('which_hacks_joined', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), null=True, size=None)), ('hack_work_starts', models.IntegerField()), ('hack_work_submits', models.IntegerField()), - ('hack_work_start_orgs', models.IntegerField()), - ('hack_work_submit_orgs', models.IntegerField()), + ('hack_work_start_orgs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), null=True, size=None)), + ('hack_work_submit_orgs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), null=True, size=None)), ('bounty_work_starts', models.IntegerField()), ('bounty_work_submits', models.IntegerField()), ('hack_started_feature', models.IntegerField()), @@ -56,30 +57,30 @@ class Migration(migrations.Migration): ('submitted_bug', models.IntegerField()), ('submitted_other', models.IntegerField()), ('submitted_improvement', models.IntegerField()), - ('bounty_earnings', models.IntegerField()), - ('bounty_work_start_orgs', models.IntegerField()), - ('bounty_work_submit_orgs', models.IntegerField()), + ('bounty_earnings', models.FloatField()), + ('bounty_work_start_orgs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), null=True, size=None)), + ('bounty_work_submit_orgs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), null=True, size=None)), ('kudos_sends', models.IntegerField()), ('kudos_receives', models.IntegerField()), ('hack_winner_kudos_received', models.IntegerField()), ('grants_opened', models.IntegerField()), ('grant_contributed', models.IntegerField()), ('grant_contributions', models.IntegerField()), - ('grant_contribution_amount', models.IntegerField()), + ('grant_contribution_amount', models.FloatField()), ('num_actions', models.IntegerField()), ('action_points', models.FloatField()), ('avg_points_per_action', models.FloatField()), - ('last_action_on', models.IntegerField()), - ('keywords', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), size=None)), - ('activity_level', models.CharField(max_length=255)), - ('reliability', models.CharField(max_length=255)), - ('average_rating', models.IntegerField()), + ('last_action_on', models.CharField(max_length=255, null=True)), + ('keywords', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), null=True, size=None)), + ('activity_level', models.CharField(max_length=255, null=True)), + ('reliability', models.CharField(max_length=255, null=True)), + ('average_rating', models.FloatField()), ('longest_streak', models.IntegerField()), - ('earnings_count', models.IntegerField()), + ('earnings_count', models.FloatField()), ('follower_count', models.IntegerField()), ('following_count', models.IntegerField()), ('num_repeated_relationships', models.IntegerField()), - ('verification_status', models.IntegerField()), + ('verification_status', models.CharField(max_length=255, null=True)), ], options={ 'managed': False, diff --git a/app/dashboard/models.py b/app/dashboard/models.py index ff194bdc5a8..d8f585fa59b 100644 --- a/app/dashboard/models.py +++ b/app/dashboard/models.py @@ -4459,21 +4459,22 @@ def get_queryset(self): class UserDirectory(models.Model): profile_id = models.CharField(max_length=255, primary_key=True) - join_date = models.CharField(max_length=255) - github_created_at = models.CharField(max_length=255) - first_name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255) + join_date = models.CharField(null=True, max_length=255) + github_created_at = models.CharField(null=True, max_length=255) + first_name = models.CharField(null=True, max_length=255) + last_name = models.CharField(null=True, max_length=255) email = models.EmailField() - handle = models.CharField(max_length=255) + handle = models.CharField(null=True, max_length=255) sms_verification = models.BooleanField() - persona = models.CharField(max_length=255) + persona = models.CharField(null=True, max_length=255) rank_coder = models.IntegerField() + rank_funder = models.IntegerField() num_hacks_joined = models.IntegerField() - which_hacks_joined = ArrayField(base_field=models.IntegerField()) + which_hacks_joined = ArrayField(base_field=models.IntegerField(), null=True) hack_work_starts = models.IntegerField() hack_work_submits = models.IntegerField() - hack_work_start_orgs = models.IntegerField() - hack_work_submit_orgs = models.IntegerField() + hack_work_start_orgs = ArrayField(base_field=models.CharField(max_length=255), null=True) + hack_work_submit_orgs = ArrayField(base_field=models.CharField(max_length=255), null=True) bounty_work_starts = models.IntegerField() bounty_work_submits = models.IntegerField() hack_started_feature = models.IntegerField() @@ -4500,30 +4501,30 @@ class UserDirectory(models.Model): submitted_bug = models.IntegerField() submitted_other = models.IntegerField() submitted_improvement = models.IntegerField() - bounty_earnings = models.IntegerField() - bounty_work_start_orgs = models.IntegerField() - bounty_work_submit_orgs = models.IntegerField() + bounty_earnings = models.FloatField() + bounty_work_start_orgs = ArrayField(base_field=models.CharField(max_length=255), null=True) + bounty_work_submit_orgs = ArrayField(base_field=models.CharField(max_length=255), null=True) kudos_sends = models.IntegerField() kudos_receives = models.IntegerField() hack_winner_kudos_received = models.IntegerField() grants_opened = models.IntegerField() grant_contributed = models.IntegerField() grant_contributions = models.IntegerField() - grant_contribution_amount = models.IntegerField() + grant_contribution_amount = models.FloatField() num_actions = models.IntegerField() action_points = models.FloatField() avg_points_per_action = models.FloatField() - last_action_on = models.IntegerField() - keywords = ArrayField(base_field=models.CharField(max_length=255)) - activity_level = models.CharField(max_length=255) - reliability = models.CharField(max_length=255) - average_rating = models.IntegerField() + last_action_on = models.CharField(null=True, max_length=255) + keywords = ArrayField(base_field=models.CharField(max_length=255), null=True) + activity_level = models.CharField(null=True, max_length=255) + reliability = models.CharField(null=True, max_length=255) + average_rating = models.FloatField() longest_streak = models.IntegerField() - earnings_count = models.IntegerField() + earnings_count = models.FloatField() follower_count = models.IntegerField() following_count = models.IntegerField() num_repeated_relationships = models.IntegerField() - verification_status = models.IntegerField() + verification_status = models.CharField(null=True, max_length=255) objects = UserDirectoryManager() diff --git a/app/dashboard/search_indexes.py b/app/dashboard/search_indexes.py index 254d7aea0f2..358d41903fa 100644 --- a/app/dashboard/search_indexes.py +++ b/app/dashboard/search_indexes.py @@ -14,12 +14,13 @@ class UserDirectoryIndex(indexes.SearchIndex, indexes.Indexable): sms_verification = indexes.BooleanField(null=True,model_attr='sms_verification',faceted=True) persona = indexes.CharField(null=True,model_attr='persona',faceted=True) rank_coder = indexes.IntegerField(null=True,model_attr='rank_coder',faceted=True) + rank_funder = indexes.IntegerField(null=True,model_attr='rank_funder',faceted=True) num_hacks_joined = indexes.IntegerField(null=True,model_attr='num_hacks_joined',faceted=True) which_hacks_joined = indexes.MultiValueField(null=True,model_attr='which_hacks_joined',faceted=True) hack_work_starts = indexes.IntegerField(null=True,model_attr='hack_work_starts',faceted=True) hack_work_submits = indexes.IntegerField(null=True,model_attr='hack_work_submits',faceted=True) - hack_work_start_orgs = indexes.IntegerField(null=True,model_attr='hack_work_start_orgs',faceted=True) - hack_work_submit_orgs = indexes.IntegerField(null=True,model_attr='hack_work_submit_orgs',faceted=True) + hack_work_start_orgs = indexes.MultiValueField(null=True,model_attr='hack_work_start_orgs',faceted=True) + hack_work_submit_orgs = indexes.MultiValueField(null=True,model_attr='hack_work_submit_orgs',faceted=True) bounty_work_starts = indexes.IntegerField(null=True,model_attr='bounty_work_starts',faceted=True) bounty_work_submits = indexes.IntegerField(null=True,model_attr='bounty_work_submits',faceted=True) hack_started_feature = indexes.IntegerField(null=True,model_attr='hack_started_feature',faceted=True) @@ -46,26 +47,26 @@ class UserDirectoryIndex(indexes.SearchIndex, indexes.Indexable): submitted_bug = indexes.IntegerField(null=True,model_attr='submitted_bug',faceted=True) submitted_other = indexes.IntegerField(null=True,model_attr='submitted_other',faceted=True) submitted_improvement = indexes.IntegerField(null=True,model_attr='submitted_improvement',faceted=True) - bounty_earnings = indexes.IntegerField(null=True,model_attr='bounty_earnings',faceted=True) - bounty_work_start_orgs = indexes.IntegerField(null=True,model_attr='bounty_work_start_orgs',faceted=True) - bounty_work_submit_orgs = indexes.IntegerField(null=True,model_attr='bounty_work_submit_orgs',faceted=True) + bounty_earnings = indexes.FloatField(null=True,model_attr='bounty_earnings',faceted=True) + bounty_work_start_orgs = indexes.MultiValueField(null=True,model_attr='bounty_work_start_orgs',faceted=True) + bounty_work_submit_orgs = indexes.MultiValueField(null=True,model_attr='bounty_work_submit_orgs',faceted=True) kudos_sends = indexes.IntegerField(null=True,model_attr='kudos_sends',faceted=True) kudos_receives = indexes.IntegerField(null=True,model_attr='kudos_receives',faceted=True) hack_winner_kudos_received = indexes.IntegerField(null=True,model_attr='hack_winner_kudos_received',faceted=True) grants_opened = indexes.IntegerField(null=True,model_attr='grants_opened',faceted=True) grant_contributed = indexes.IntegerField(null=True,model_attr='grant_contributed',faceted=True) grant_contributions = indexes.IntegerField(null=True,model_attr='grant_contributions',faceted=True) - grant_contribution_amount = indexes.IntegerField(null=True,model_attr='grant_contribution_amount',faceted=True) + grant_contribution_amount = indexes.FloatField(null=True,model_attr='grant_contribution_amount',faceted=True) num_actions = indexes.IntegerField(null=True,model_attr='num_actions',faceted=True) action_points = indexes.IntegerField(null=True,model_attr='action_points',faceted=True) - avg_points_per_action = indexes.IntegerField(null=True,model_attr='avg_points_per_action',faceted=True) + avg_points_per_action = indexes.FloatField(null=True,model_attr='avg_points_per_action',faceted=True) last_action_on = indexes.CharField(null=True,model_attr='last_action_on') keywords = indexes.MultiValueField(null=True,model_attr='keywords',faceted=True) activity_level = indexes.CharField(null=True,model_attr='activity_level',faceted=True) reliability = indexes.CharField(null=True,model_attr='reliability',faceted=True) - average_rating = indexes.IntegerField(null=True,model_attr='average_rating',faceted=True) + average_rating = indexes.FloatField(null=True,model_attr='average_rating',faceted=True) longest_streak = indexes.IntegerField(null=True,model_attr='longest_streak',faceted=True) - earnings_count = indexes.IntegerField(null=True,model_attr='earnings_count',faceted=True) + earnings_count = indexes.FloatField(null=True,model_attr='earnings_count',faceted=True) follower_count = indexes.IntegerField(null=True,model_attr='follower_count',faceted=True) following_count = indexes.IntegerField(null=True,model_attr='following_count',faceted=True) num_repeated_relationships = indexes.IntegerField(null=True,model_attr='num_repeated_relationships',faceted=True) diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 4c14b63898a..3a69bd36920 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -877,7 +877,7 @@ def user_lookup(request): return HttpResponse(status=404) path = request.get_full_path().replace('/user_lookup', '') - remote_url = f'{"https" if not settings.DEBUG else "http"}://{settings.ELASTIC_SEARCH_URL}:9200{path}' + remote_url = f'{settings.HAYSTACK_ELASTIC_SEARCH_URL}{path}' from proxy.views import proxy_view return proxy_view(request, remote_url) diff --git a/docker-compose.yml b/docker-compose.yml index f3ab4475230..1f2fcaeed28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: memory: 128M elasticsearch: - image: launcher.gcr.io/google/elasticsearch2 + image: launcher.gcr.io/google/elasticsearch5 ports: - "9200:9200" - "9300:9300"