Skip to content

Commit

Permalink
User Directory: Elastic Search Edition. (#7204)
Browse files Browse the repository at this point in the history
* WIP, Frank's Super Query added as a view, connected to a ViewSet and returned through the api

* WIP: Elastic Search using Haystack
- indexes all Profiles, mergeing the to_dict method into the search store itself,
- Bounty, and HackathonProject indexers are stubbed out to work on a path for relational models

* WIP urls and search view

* enable auto complete for lastname, handle, first name, and persona

* remove other indexers

* WIP: UserDirectory
 - vue search app hacked into the old user search application
 - dynamic filters, are all set to checkbox filters presently adjusting by type of metrics is required

* wip, user cards rendering

* dynamic auto complete based on fields like kibana, still WIP

* WIP: dynamic filtering, some style work still needed, dynamic query building is possible

* numeric list filter adjusted to account for elastic type

* V0 RC1

* restore old user directory

* restored docker-compose, addressed review comments

* remove email from the elastic search indexer

* add crontab entry for elasticsearch update

* haystack settings configured to retrieve the proper env variable, removing stale ui components
  • Loading branch information
androolloyd authored Aug 31, 2020
1 parent 6c7d6cc commit 9581fdb
Show file tree
Hide file tree
Showing 19 changed files with 1,369 additions and 8 deletions.
3 changes: 2 additions & 1 deletion app/app/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def preprocess(request):
chat_url = get_chat_url(front_end=True)
chat_access_token = ''
chat_id = ''

search_url = '';
user_is_authenticated = request.user.is_authenticated
profile = request.user.profile if user_is_authenticated and hasattr(request.user, 'profile') else None
if user_is_authenticated and profile and profile.pk:
Expand Down Expand Up @@ -134,6 +134,7 @@ def preprocess(request):
'MEDIA_URL': settings.MEDIA_URL,
'max_length': max_length,
'max_length_offset': max_length_offset,
'search_url': settings.ELASTIC_SEARCH_LB_URL,
'chat_url': chat_url,
'base_url': settings.BASE_URL,
'chat_id': chat_id,
Expand Down
12 changes: 12 additions & 0 deletions app/app/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@
'wiki.plugins.macros.apps.MacrosConfig',
'adminsortable2',
'debug_toolbar',
'haystack',
]

MIDDLEWARE = [
Expand Down Expand Up @@ -822,6 +823,17 @@ def callback(request):


ELASTIC_SEARCH_URL = env('ELASTIC_SEARCH_URL', default='')
ELASTIC_SEARCH_LB_URL = env('ELASTIC_SEARCH_LB_URL', default='')

HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine',
'URL': f"{ELASTIC_SEARCH_URL}:9200",
'INDEX_NAME': 'haystack',
},
}
# Update Search index in realtime (using models.db.signals)
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

account_sid = env('TWILIO_ACCOUNT_SID', default='')
auth_token = env('TWILIO_AUTH_TOKEN', default='')
Expand Down
2 changes: 2 additions & 0 deletions app/app/templates/search/indexes/dashboard/bounty_text.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{{ object.title }}

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{{ object.title }}
{{ object.description }}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{{ object.object }}
172 changes: 172 additions & 0 deletions app/app/templates/search/search.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
{% comment %}
Copyright (C) 2020 Gitcoin Core

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

{% endcomment %}
{% load i18n static email_obfuscator add_url_schema avatar_tags %}
<!DOCTYPE html>
<html lang="en">

<head>
{% include 'shared/head.html' %}
{% include 'shared/cards.html' %}
<link rel="stylesheet" href={% static "v2/css/users.css" %} />
<link rel="stylesheet" href={% static "v2/css/tag.css" %} />
<link rel="stylesheet" href={% static "v2/css/rating.css" %} />
<link rel="stylesheet" href="https://unpkg.com/vue-innersearch/default-innersearch-theme.min.css">
</head>

<body id="{{ explore }}" class="interior {{active}} g-font-muli">
{% include 'shared/tag_manager_2.html' %}
<div class="container-fluid header dash">
{% include 'shared/top_nav.html' with class='d-md-flex' %}
{% include 'home/nav.html' %}
</div>
{% block content %}
<h2>Search</h2>

<form method="get" action="">
<table>
{{ form.as_table }}
<tr>
<td>&nbsp;</td>
<td>
<input type="submit" value="Search">
</td>
</tr>
</table>

{% if query %}
<h3>Results</h3>

{% for result in page.object_list %}
<p>
<a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a>
</p>
{% empty %}
<p>No results found.</p>
{% endfor %}

{% if page.has_previous or page.has_next %}
<div>
{% if page.has_previous %}<a href="?q={{ query }}&amp;page={{ page.previous_page_number }}">{% endif %}&laquo; Previous{% if page.has_previous %}</a>{% endif %}
|
{% if page.has_next %}<a href="?q={{ query }}&amp;page={{ page.next_page_number }}">{% endif %}Next &raquo;{% if page.has_next %}</a>{% endif %}
</div>
{% endif %}
{% else %}
{# Show some example queries to run, maybe query syntax, something else? #}
{% endif %}
</form>
{% endblock %}
<div id="gc-users-directory" v-cloak>

<section>
<h1 class='is-title'>Hacky User Directory</h1>

<hr class='is-line' />

<div class="filters" style="width : 90%; margin : 0 auto;">
<tag-filter :for="'searchbox'"></tag-filter>
<tag-filter :for="'searchdatalist'">
<template slot-scope="{ data }">
[[ data._source.keywords ]]
</template>
</tag-filter>
<tag-filter :for="'nlf'">
<template slot-scope="{ data }">
Number of Hacks Joined: [[ data._source.keywords ]]
</template>
</tag-filter>
</div>

<div class="is-columns">
<div class="is-column is-one-fifth">
<div>
<numeric-list-filter :id="'nlf'" :field="'num_hacks_joined'">
<template slot="header">
<h3 class="is-nlf-title">Number of Hacks Joined : </h3>
</template>
</numeric-list-filter>
</div>
</div>
<div class="is-column">
<div>
<searchbox :id="'searchbox'" :autofocus="true" :realtime="true" :timeout="200" :field="['keywords', 'handle', 'persona']" :placeholder="'Search by keyword'"></searchbox>
<search-datalist :id="'searchdatalist'" :realtime="true" :field="'keywords'" :suggestion="['keywords']">
<template slot="items" slot-scope="{ item, value }">
[[ value ]] [[ item._source.keywords ]]
</template>

<template slot="nosuggestion" slot-scope="{ value }">
Sorry, "[[ value ]]" doesn't exist... :(
</template>

<template slot="suggestions" slot-scope="{ suggestion }">
<span v-html="suggestion.highlight.first_name ? suggestion.highlight.first_name[0] : suggestion._source.first_name"></span>
<span v-html="suggestion.highlight.last_name ? suggestion.highlight.last_name[0] : suggestion._source.last_name"></span>
<span v-html="suggestion.highlight.handle ? suggestion.highlight.handle[0] : suggestion._source.handle"></span>
<span v-html="suggestion.highlight.persona ? suggestion.highlight.persona[0] : suggestion._source.persona"></span>
</template>
</search-datalist>

<div style="margin: 20px auto;width: 90%">
<search-button></search-button>
<reset-button :emptyHits="false"></reset-button>
</div>
<hits>
<template slot="hits" slot-scope="{ hits }">
<div class="is-score is-hits">
<strong v-if="hits.score === 0">No result found</strong>
<strong v-else-if="hits.score === 1">1 result found</strong>
<strong v-else-if="hits.score > 1">[[ hits.score ]] results found</strong>
</div>
<div v-for="item in hits.items" :item="item">
<div><strong>Identity (firstname, lastname) :</strong> ([[ item._source.first_name ]] [[ item._source.last_name ]] )</div>
</div>
</template>
</hits>

<paginate :previousText="'&#x2B9C; Previous page'" :nextText="'Next page &#x2B9E;'" :size="10"></paginate>
</div>
</div>
</div>
</section>
</div>

<script type="text/x-template" id="select2-template">
<select>
<slot></slot>
</select>
</script>

{% csrf_token %}
{% include 'shared/analytics.html' %}
{% include 'shared/footer_scripts.html' %}
{% include 'shared/footer.html' %}
<script src="https://unpkg.com/[email protected]/vue-innersearch.min.js"></script>


<script>
$('body').bootstrapTooltip({
selector: '[data-toggle="tooltip"]'
});
const csrftoken = jQuery("[name=csrfmiddlewaretoken]").val();
document.keywords = {{keywords | safe}};
</script>
<script src="{% static "v2/js/vue-components.js" %}"></script>
<script src="{% static "v2/js/users.js" %}"></script>
</body>
</html>
3 changes: 2 additions & 1 deletion app/app/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@

# User Directory
re_path(r'^users/?', dashboard.views.users_directory, name='users_directory'),
re_path(r'^user_directory/?', dashboard.views.users_directory_elastic, name='users_directory_elastic'),
re_path(r'^tribes/explore', dashboard.views.users_directory, name='tribes_directory'),

# Alpha functionality
Expand Down Expand Up @@ -501,7 +502,6 @@
bounty_requests.views.update_bounty_request_v1,
name='update_bounty_request_v1'
),

# admin views
re_path(r'^_administration/?', admin.site.urls, name='admin'),
path(
Expand Down Expand Up @@ -708,6 +708,7 @@

# users
url(r'^api/v0.1/user_bounties/', dashboard.views.get_user_bounties, name='get_user_bounties'),
url(r'^api/v0.1/users_csv/', dashboard.views.output_users_to_csv, name='users_csv'),
url(r'^api/v0.1/bounty_mentor/', dashboard.views.bounty_mentor, name='bounty_mentor'),
url(r'^api/v0.1/users_fetch/', dashboard.views.users_fetch, name='users_fetch'),

Expand Down
Loading

0 comments on commit 9581fdb

Please sign in to comment.