puts searchresults objects in elasticsearch/ makes search GA (gitcoin…

…co#6340)

* POC

* puts documents in elasticsearch

* make fix

* fix typos

Co-authored-by: Dan Lipert <[email protected]>

app/app/settings.py

@@ -809,3 +809,5 @@ def callback(request):
 AVATAR_ADDRESS = env('AVATAR_ADDRESS', default='0x00De4B13153673BCAE2616b67bf822500d325Fc3')
 AVATAR_PRIVATE_KEY = env('AVATAR_PRIVATE_KEY', default='0x00De4B13153673BCAE2616b67bf822500d325Fc3')
 
+
+ELASTIC_SEARCH_URL = env('ELASTIC_SEARCH_URL', default='')

app/dashboard/templates/shared/nav_auth.html

@@ -17,9 +17,7 @@
 {% load i18n static %}
 {% load kudos_extras %}
 <input id="is-authenticated" type="hidden" value={{ user.is_authenticated }}>
-{% if is_alpha_tester %}
-  <div id=search_container class=""></div>
-{% endif %}
+<div id=search_container class=""></div>
 {% if user.is_authenticated %}
   <div class="nav-item dropdown interior">
     <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button"

app/search/management/commands/create_search_index.py

@@ -0,0 +1,30 @@
+'''
+    Copyright (C) 2019 Gitcoin Core
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+'''
+from django.core.management.base import BaseCommand
+
+from search.models import SearchResult
+
+
+class Command(BaseCommand):
+
+    help = 'uploads search results into elasticsearch'
+
+    def handle(self, *args, **options):
+        for sr in SearchResult.objects.all():
+            print(sr.pk)
+            sr.put_on_elasticsearch()

app/search/models.py

@@ -1,8 +1,11 @@
+from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
+from django.utils import timezone
 
 from economy.models import SuperModel
+from elasticsearch import Elasticsearch
 
 
 class SearchResult(SuperModel):
@@ -21,6 +24,40 @@ def __str__(self):
         return f"{self.source_type}; {self.url}"
 
 
+    def put_on_elasticsearch(self):
+        if self.visible_to:
+            return None
+
+        es = Elasticsearch([settings.ELASTIC_SEARCH_URL])
+        source_type  = str(str(self.source_type).replace('token', 'kudos')).title()
+        full_search = f"{self.title}{self.description}{source_type}"
+        doc = {
+            'title': self.title,
+            'description': self.description,
+            'full_search': full_search,
+            'url': self.url,
+            'pk': self.pk,
+            'img_url': self.img_url,
+            'timestamp': timezone.now(),
+            'source_type': source_type,
+        }
+        res = es.index(index="search-index", id=self.pk, body=doc)
+
+
+def search(query):
+    if not settings.ELASTIC_SEARCH_URL:
+        return {}
+    es = Elasticsearch([settings.ELASTIC_SEARCH_URL])
+    res = es.search(index="search-index", body={
+      "query": {
+        "match": {
+          "full_search": query,
+        }
+      }
+    })
+    return res
+
+
 class ProgrammingLanguage(SuperModel):
     """Records ProgrammingLanguage - the list for programming langauges"""
     val = models.CharField(max_length=255, default='')

app/search/views.py

@@ -8,12 +8,22 @@
 from ratelimit.decorators import ratelimit
 from retail.helpers import get_ip
 
-from .models import SearchResult
+from .models import SearchResult, search
 
 
 @ratelimit(key='ip', rate='30/m', method=ratelimit.UNSAFE, block=True)
 def search(request):
     keyword = request.GET.get('term', '')
+
+    # attempt elasticsearch first
+    try:
+        all_result_sets = search(keyword)  
+        return_results = [ele['_source'] for ele in all_result_sets['hits']['hits']]
+        mimetype = 'application/json'
+        return HttpResponse(json.dumps(return_results), mimetype)
+    except Exception as e:
+        print(e)
+
     all_result_sets = [SearchResult.objects.filter(title__icontains=keyword), SearchResult.objects.filter(description__icontains=keyword)]
     return_results = []
     exclude_pks = []

requirements/base.txt

@@ -93,4 +93,5 @@ redis==3.3.11
 pandas
 wiki
 django-bulk-update
+elasticsearch
 

scripts/crontab

@@ -81,6 +81,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/us
 00 10 * * 5 cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash post_leaderboard_to_slack  >> /var/log/gitcoin/post_leaderboard_to_slack.log  2>&1
 0 9 * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash no_applicants_email  >> /var/log/gitcoin/no_applicants_email.log 2>&1
 0 10 * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash pull_grant_twitter_info  >> /var/log/gitcoin/pull_grant_twitter_info.log 2>&1
+0 11 * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash create_search_index  >> /var/log/gitcoin/create_search_index.log 2>&1
 
 
 ## PERFTOOLS