From 9581fdb995612685063d91f5766a269743461874 Mon Sep 17 00:00:00 2001 From: Andrew Redden Date: Mon, 31 Aug 2020 09:43:13 -0300 Subject: [PATCH] User Directory: Elastic Search Edition. (#7204) * WIP, Frank's Super Query added as a view, connected to a ViewSet and returned through the api * WIP: Elastic Search using Haystack - indexes all Profiles, mergeing the to_dict method into the search store itself, - Bounty, and HackathonProject indexers are stubbed out to work on a path for relational models * WIP urls and search view * enable auto complete for lastname, handle, first name, and persona * remove other indexers * WIP: UserDirectory - vue search app hacked into the old user search application - dynamic filters, are all set to checkbox filters presently adjusting by type of metrics is required * wip, user cards rendering * dynamic auto complete based on fields like kibana, still WIP * WIP: dynamic filtering, some style work still needed, dynamic query building is possible * numeric list filter adjusted to account for elastic type * V0 RC1 * restore old user directory * restored docker-compose, addressed review comments * remove email from the elastic search indexer * add crontab entry for elasticsearch update * haystack settings configured to retrieve the proper env variable, removing stale ui components --- app/app/context.py | 3 +- app/app/settings.py | 12 + .../search/indexes/dashboard/bounty_text.txt | 2 + .../dashboard/hackathonproject_text.txt | 2 + .../indexes/dashboard/userdirectory_text.txt | 1 + app/app/templates/search/search.html | 172 +++++ app/app/urls.py | 3 +- app/assets/v2/js/users-elastic.js | 632 ++++++++++++++++++ app/assets/v2/js/vue-components.js | 7 +- app/dashboard/models.py | 84 +++ app/dashboard/router.py | 12 +- app/dashboard/search_indexes.py | 85 +++ .../templates/dashboard/users-elastic.html | 287 ++++++++ app/dashboard/views.py | 53 +- .../templates/shared/footer_scripts.html | 2 + docker-compose.yml | 12 +- elasticsearch.yml | 3 + requirements/base.txt | 4 +- scripts/crontab | 1 + 19 files changed, 1369 insertions(+), 8 deletions(-) create mode 100644 app/app/templates/search/indexes/dashboard/bounty_text.txt create mode 100644 app/app/templates/search/indexes/dashboard/hackathonproject_text.txt create mode 100644 app/app/templates/search/indexes/dashboard/userdirectory_text.txt create mode 100644 app/app/templates/search/search.html create mode 100644 app/assets/v2/js/users-elastic.js create mode 100644 app/dashboard/search_indexes.py create mode 100644 app/dashboard/templates/dashboard/users-elastic.html create mode 100644 elasticsearch.yml diff --git a/app/app/context.py b/app/app/context.py index 936ba48acd6..cad2a752c82 100644 --- a/app/app/context.py +++ b/app/app/context.py @@ -74,7 +74,7 @@ def preprocess(request): chat_url = get_chat_url(front_end=True) chat_access_token = '' chat_id = '' - + search_url = ''; user_is_authenticated = request.user.is_authenticated profile = request.user.profile if user_is_authenticated and hasattr(request.user, 'profile') else None if user_is_authenticated and profile and profile.pk: @@ -134,6 +134,7 @@ def preprocess(request): 'MEDIA_URL': settings.MEDIA_URL, 'max_length': max_length, 'max_length_offset': max_length_offset, + 'search_url': settings.ELASTIC_SEARCH_LB_URL, 'chat_url': chat_url, 'base_url': settings.BASE_URL, 'chat_id': chat_id, diff --git a/app/app/settings.py b/app/app/settings.py index b5f1915c12b..a70c9ae8599 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -146,6 +146,7 @@ 'wiki.plugins.macros.apps.MacrosConfig', 'adminsortable2', 'debug_toolbar', + 'haystack', ] MIDDLEWARE = [ @@ -822,6 +823,17 @@ def callback(request): ELASTIC_SEARCH_URL = env('ELASTIC_SEARCH_URL', default='') +ELASTIC_SEARCH_LB_URL = env('ELASTIC_SEARCH_LB_URL', default='') + +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', + 'URL': f"{ELASTIC_SEARCH_URL}:9200", + 'INDEX_NAME': 'haystack', + }, +} +# Update Search index in realtime (using models.db.signals) +HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' account_sid = env('TWILIO_ACCOUNT_SID', default='') auth_token = env('TWILIO_AUTH_TOKEN', default='') diff --git a/app/app/templates/search/indexes/dashboard/bounty_text.txt b/app/app/templates/search/indexes/dashboard/bounty_text.txt new file mode 100644 index 00000000000..e21f52a6d2d --- /dev/null +++ b/app/app/templates/search/indexes/dashboard/bounty_text.txt @@ -0,0 +1,2 @@ +{{ object.title }} + diff --git a/app/app/templates/search/indexes/dashboard/hackathonproject_text.txt b/app/app/templates/search/indexes/dashboard/hackathonproject_text.txt new file mode 100644 index 00000000000..37c5aa0fa10 --- /dev/null +++ b/app/app/templates/search/indexes/dashboard/hackathonproject_text.txt @@ -0,0 +1,2 @@ +{{ object.title }} +{{ object.description }} diff --git a/app/app/templates/search/indexes/dashboard/userdirectory_text.txt b/app/app/templates/search/indexes/dashboard/userdirectory_text.txt new file mode 100644 index 00000000000..1fe45381576 --- /dev/null +++ b/app/app/templates/search/indexes/dashboard/userdirectory_text.txt @@ -0,0 +1 @@ +{{ object.object }} diff --git a/app/app/templates/search/search.html b/app/app/templates/search/search.html new file mode 100644 index 00000000000..de0f53e41dc --- /dev/null +++ b/app/app/templates/search/search.html @@ -0,0 +1,172 @@ +{% comment %} + Copyright (C) 2020 Gitcoin Core + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +{% endcomment %} +{% load i18n static email_obfuscator add_url_schema avatar_tags %} + + + + + {% include 'shared/head.html' %} + {% include 'shared/cards.html' %} + + + + + + + + {% include 'shared/tag_manager_2.html' %} +
+ {% include 'shared/top_nav.html' with class='d-md-flex' %} + {% include 'home/nav.html' %} +
+ {% block content %} +

Search

+ +
+ + {{ form.as_table }} + + + + +
  + +
+ + {% if query %} +

Results

+ + {% for result in page.object_list %} +

+ {{ result.object.title }} +

+ {% empty %} +

No results found.

+ {% endfor %} + + {% if page.has_previous or page.has_next %} +
+ {% if page.has_previous %}{% endif %}« Previous{% if page.has_previous %}{% endif %} + | + {% if page.has_next %}{% endif %}Next »{% if page.has_next %}{% endif %} +
+ {% endif %} + {% else %} + {# Show some example queries to run, maybe query syntax, something else? #} + {% endif %} +
+{% endblock %} +
+ +
+

Hacky User Directory

+ +
+ +
+ + + + + + + +
+ +
+
+
+ + + +
+
+
+
+ + + + + + + + + +
+ + +
+ + + + + +
+
+
+
+
+ + + + {% csrf_token %} + {% include 'shared/analytics.html' %} + {% include 'shared/footer_scripts.html' %} + {% include 'shared/footer.html' %} + + + + + + + + diff --git a/app/app/urls.py b/app/app/urls.py index 3df2be4f80b..553c6abd377 100644 --- a/app/app/urls.py +++ b/app/app/urls.py @@ -370,6 +370,7 @@ # User Directory re_path(r'^users/?', dashboard.views.users_directory, name='users_directory'), + re_path(r'^user_directory/?', dashboard.views.users_directory_elastic, name='users_directory_elastic'), re_path(r'^tribes/explore', dashboard.views.users_directory, name='tribes_directory'), # Alpha functionality @@ -501,7 +502,6 @@ bounty_requests.views.update_bounty_request_v1, name='update_bounty_request_v1' ), - # admin views re_path(r'^_administration/?', admin.site.urls, name='admin'), path( @@ -708,6 +708,7 @@ # users url(r'^api/v0.1/user_bounties/', dashboard.views.get_user_bounties, name='get_user_bounties'), + url(r'^api/v0.1/users_csv/', dashboard.views.output_users_to_csv, name='users_csv'), url(r'^api/v0.1/bounty_mentor/', dashboard.views.bounty_mentor, name='bounty_mentor'), url(r'^api/v0.1/users_fetch/', dashboard.views.users_fetch, name='users_fetch'), diff --git a/app/assets/v2/js/users-elastic.js b/app/assets/v2/js/users-elastic.js new file mode 100644 index 00000000000..633d1dfb0ca --- /dev/null +++ b/app/assets/v2/js/users-elastic.js @@ -0,0 +1,632 @@ +let users = []; +let usersPage = 1; +let usersNumPages = ''; +let usersHasNext = false; +let numUsers = ''; +let hackathonId = document.hasOwnProperty('hackathon_id') ? document.hackathon_id : ''; +// let funderBounties = []; + +Vue.mixin({ + methods: { + messageUser: function(handle) { + let vm = this; + const url = handle ? `${vm.chatURL}/hackathons/messages/@${handle}` : `${vm.chatURL}/`; + + chatWindow = window.open(url, 'Loading', 'top=0,left=0,width=400,height=600,status=no,toolbar=no,location=no,menubar=no,titlebar=no'); + }, + + fetchUsers: function(newPage) { + let vm = this; + + vm.isLoading = true; + vm.noResults = false; + + if (newPage) { + vm.usersPage = newPage; + } + vm.params.page = vm.usersPage; + if (hackathonId) { + vm.params.hackathon = hackathonId; + } + if (vm.searchTerm) { + vm.params.search = vm.searchTerm; + } else { + delete vm.params['search']; + } + + if (vm.hideFilterButton) { + vm.params.persona = 'tribe'; + } + + if (vm.params.persona === 'tribe') { + // remove filters which do not apply for tribes directory + delete vm.params['rating']; + delete vm.params['organisation']; + delete vm.params['skills']; + } + + if (vm.tribeFilter) { + vm.params.tribe = vm.tribeFilter; + } + + + let searchParams = new URLSearchParams(vm.params); + + let apiUrlUsers = `/api/v0.1/users_fetch/?${searchParams.toString()}`; + + if (vm.hideFilterButton) { + apiUrlUsers += '&type=explore_tribes'; + } + + var getUsers = fetchData(apiUrlUsers, 'GET'); + + $.when(getUsers).then(function(response) { + + response.data.forEach(function(item) { + vm.users.push(item); + }); + + vm.usersNumPages = response.num_pages; + vm.usersHasNext = response.has_next; + vm.numUsers = response.count; + vm.showBanner = response.show_banner; + vm.persona = response.persona; + vm.rating = response.rating; + if (vm.usersHasNext) { + vm.usersPage = ++vm.usersPage; + + } else { + vm.usersPage = 1; + } + + if (vm.users.length) { + vm.noResults = false; + } else { + vm.noResults = true; + } + vm.isLoading = false; + }); + }, + searchUsers: function() { + let vm = this; + + vm.users = []; + + vm.fetchUsers(1); + + }, + bottomVisible: function() { + let vm = this; + + const scrollY = window.scrollY; + const visible = document.documentElement.clientHeight; + const pageHeight = document.documentElement.scrollHeight - 500; + const bottomOfPage = visible + scrollY >= pageHeight; + + if (bottomOfPage || pageHeight < visible) { + if (vm.usersHasNext) { + vm.fetchUsers(); + vm.usersHasNext = false; + } + } + }, + fetchBounties: function() { + let vm = this; + + // fetch bounties + let apiUrlBounties = '/api/v0.1/user_bounties/'; + + let getBounties = fetchData(apiUrlBounties, 'GET'); + + $.when(getBounties).then((response) => { + vm.isFunder = response.is_funder; + vm.funderBounties = response.data; + }); + + }, + openBounties: function(user) { + let vm = this; + + vm.userSelected = user; + }, + sendInvite: function(bounty, user) { + let vm = this; + + console.log(vm.bountySelected, bounty, user, csrftoken); + let apiUrlInvite = '/api/v0.1/social_contribution_email/'; + let postInvite = fetchData( + apiUrlInvite, + 'POST', + {'usersId': [user], 'bountyId': bounty.id}, + {'X-CSRFToken': csrftoken} + ); + + $.when(postInvite).then((response) => { + console.log(response); + if (response.status === 500) { + _alert(response.msg, 'error'); + + } else { + vm.$refs['user-modal'].closeModal(); + _alert('The invitation has been sent', 'info'); + } + }); + }, + sendInviteAll: function(bountyUrl) { + let vm = this; + const apiUrlInvite = '/api/v0.1/bulk_invite/'; + const postInvite = fetchData( + apiUrlInvite, + 'POST', + {'params': vm.params, 'bountyId': bountyUrl}, + {'X-CSRFToken': csrftoken} + ); + + $.when(postInvite).then((response) => { + console.log(response); + if (response.status !== 200) { + _alert(response.msg, 'error'); + + } else { + vm.$refs['user-modal'].closeModal(); + _alert('The invitation has been sent', 'info'); + } + }); + + }, + getIssueDetails: function(url) { + let vm = this; + const apiUrldetails = `/actions/api/v0.1/bounties/?github_url=${encodeURIComponent(url)}`; + + vm.errorIssueDetails = undefined; + + if (url.indexOf('github.com/') < 0) { + vm.issueDetails = null; + vm.errorIssueDetails = 'Please paste a github issue url'; + return; + } + vm.issueDetails = undefined; + const getIssue = fetchData(apiUrldetails, 'GET'); + + $.when(getIssue).then((response) => { + if (response[0]) { + vm.issueDetails = response[0]; + vm.errorIssueDetails = undefined; + } else { + vm.issueDetails = null; + vm.errorIssueDetails = 'This issue wasn\'t bountied yet.'; + } + }); + + }, + closeModal() { + this.$refs['user-modal'].closeModal(); + }, + inviteOnMount: function() { + let vm = this; + + vm.contributorInvite = getURLParams('invite'); + vm.currentBounty = getURLParams('current-bounty'); + + if (vm.contributorInvite) { + let api = `/api/v0.1/users_fetch/?search=${vm.contributorInvite}`; + let getUsers = fetchData(api, 'GET'); + + $.when(getUsers).then(function(response) { + if (response && response.data.length) { + vm.openBounties(response.data[0]); + $('#userModal').bootstrapModal('show'); + } else { + _alert('The user was not found. Please try using the search box.', 'error'); + } + }); + } + }, + extractURLFilters: function() { + let vm = this; + let params = getURLParams(); + + vm.users = []; + + if (params) { + for (var prop in params) { + if (prop === 'skills') { + vm.$set(vm.params, prop, params[prop].split(',')); + } else { + vm.$set(vm.params, prop, params[prop]); + } + } + } + }, + joinTribe: function(user, event) { + event.target.disabled = true; + const url = `/tribe/${user.handle}/join/`; + const sendJoin = fetchData(url, 'POST', {}, {'X-CSRFToken': csrftoken}); + + $.when(sendJoin).then(function(response) { + event.target.disabled = false; + + if (response.is_member) { + ++user.follower_count; + user.is_following = true; + } else { + --user.follower_count; + user.is_following = false; + } + + event.target.classList.toggle('btn-outline-green'); + event.target.classList.toggle('btn-gc-blue'); + }).fail(function(error) { + event.target.disabled = false; + }); + } + } +}); +Vue = Vue.extend({ + delimiters: [ '[[', ']]' ] +}); + + +Vue.component('directory-card', { + name: 'DirectoryCard', + delimiters: [ '[[', ']]' ], + props: [ 'user', 'funderBounties' ] +}); +Vue.use(innerSearch.default); +Vue.component('autocomplete', { + props: [ 'options', 'value' ], + template: '#select2-template', + methods: { + formatMapping: function(item) { + console.log(item); + return item.name; + }, + formatMappingSelection: function(filter) { + return ''; + } + }, + mounted() { + let count = 0; + let vm = this; + + let data = $.map(this.options, function(obj, key) { + obj.id = count++; + obj.text = key; + return obj; + }); + + + $(vm.$el).select2({ + data: data, + multiple: true, + allowClear: true, + placeholder: 'Search for another filter to add', + minimumInputLength: 1, + escapeMarkup: function(markup) { + return markup; + } + }) + .on('change', function() { + console.log('changed'); + let val = $(vm.$el).val(); + + let changeData = $.map(val, function(filter) { + return data[filter]; + }); + + vm.$emit('input', changeData); + }); + + // fix for wrong position on select open + var select2Instance = $(vm.$el).data('select2'); + + select2Instance.on('results:message', function(params) { + this.dropdown._resizeDropdown(); + this.dropdown._positionDropdown(); + }); + }, + destroyed: function() { + $(this.$el).off().select2('destroy'); + this.$emit('destroyed'); + } +}); +Vue.component('user-directory', { + delimiters: [ '[[', ']]' ], + props: [ 'tribe', 'is_my_org' ], + data: function() { + return { + orgOwner: this.is_my_org || false, + userFilter: { + options: [ + {text: 'All', value: 'all'}, + {text: 'Tribe Owners', value: 'owners'}, + {text: 'Tribe Members', value: 'members'}, + {text: 'Tribe Hackers', value: 'hackers'} + ] + }, + tribeFilter: this.tribe || '', + users, + usersPage, + hackathonId, + usersNumPages, + usersHasNext, + numUsers, + media_url, + chatURL: document.chatURL || 'https://chat.gitcoin.co/', + searchTerm: null, + bottom: false, + params: { + 'user_filter': 'all' + }, + funderBounties: [], + currentBounty: undefined, + contributorInvite: undefined, + isFunder: false, + bountySelected: null, + userSelected: [], + showModal: false, + showFilters: true, + skills: document.keywords, + selectedSkills: [], + noResults: false, + isLoading: true, + gitcoinIssueUrl: '', + issueDetails: undefined, + errorIssueDetails: undefined, + showBanner: undefined, + persona: undefined, + hideFilterButton: !!document.getElementById('explore_tribes'), + expandFilter: true + }; + }, + + mounted() { + this.fetchUsers(); + this.tribeFilter = this.tribe; + this.$watch('params', function(newVal, oldVal) { + this.searchUsers(); + }, { + deep: true + }); + }, + created() { + if (document.contxt.github_handle && this.is_my_org) { + this.fetchBounties(); + } + this.inviteOnMount(); + this.extractURLFilters(); + }, + beforeMount() { + if (this.isMobile) { + this.showFilters = false; + } + window.addEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }, false); + }, + beforeDestroy() { + window.removeEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }); + } +}); +Vue.component('user-directory-elastic', { + delimiters: [ '[[', ']]' ], + data: function() { + return { + filters: [], + esColumns: [], + filterLoaded: false, + users, + usersPage, + usersNumPages, + usersHasNext, + numUsers, + media_url, + chatURL: document.chatURL || 'https://chat.gitcoin.co/', + searchTerm: null, + bottom: false, + params: {}, + funderBounties: [], + currentBounty: undefined, + contributorInvite: undefined, + isFunder: false, + bountySelected: null, + userSelected: [], + showModal: false, + showFilters: !document.getElementById('explore_tribes'), + skills: document.keywords, + selectedSkills: [], + noResults: false, + isLoading: true, + gitcoinIssueUrl: '', + issueDetails: undefined, + errorIssueDetails: undefined, + showBanner: undefined, + persona: undefined, + hideFilterButton: !!document.getElementById('explore_tribes') + }; + }, + methods: { + autoCompleteDestroyed: function() { + this.filters = []; + }, + autoCompleteChange: function(filters) { + this.filters = filters; + }, + outputToCSV: function() { + + let output = []; + + $.map(this.items, function(obj, key) { + output.push(obj.profile_id); + }); + + let url = '/api/v0.1/users_csv/'; + + $.get(url, {profile_ids: output}) + .then(resp => resp.blob()) + .then(blob => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + + a.style.display = 'none'; + a.href = url; + a.download = `users_csv-${Date.now()}.json`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + _alert('File download complete'); + }) + .catch(() => _alert('There was an issue downloading your file')); + }, + fetchMappings: function() { + let vm = this; + + $.when(vm.header.client.indices.getMapping()) + .then(response => { + vm.esColumns = response[vm.header.index]['mappings'][vm.header.type]['properties']; + vm.filterLoaded = true; + }); + } + }, + mounted() { + this.fetchMappings(); + // this.fetchUsers(); + this.$watch('params', function(newVal, oldVal) { + this.searchUsers(); + }, { + deep: true + }); + }, + created() { + this.setHost('https://elastic.androolloyd.com'); // TODO: set to proper env variable + this.setIndex('haystack'); + this.setType('modelresult'); + this.fetchBounties(); + this.inviteOnMount(); + this.extractURLFilters(); + }, + beforeMount() { + window.addEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }, false); + }, + beforeDestroy() { + window.removeEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }); + } +}); +if (document.getElementById('gc-users-directory')) { + + window.UserDirectory = new Vue({ + delimiters: [ '[[', ']]' ], + el: '#gc-users-directory', + data: { + filters: [], + esColumns: [], + filterLoaded: false, + users, + usersPage, + usersNumPages, + usersHasNext, + numUsers, + media_url, + chatURL: document.chatURL || 'https://chat.gitcoin.co/', + searchTerm: null, + bottom: false, + params: {}, + funderBounties: [], + currentBounty: undefined, + contributorInvite: undefined, + isFunder: false, + bountySelected: null, + userSelected: [], + showModal: false, + showFilters: !document.getElementById('explore_tribes'), + skills: document.keywords, + selectedSkills: [], + noResults: false, + isLoading: true, + gitcoinIssueUrl: '', + issueDetails: undefined, + errorIssueDetails: undefined, + showBanner: undefined, + persona: undefined, + hideFilterButton: !!document.getElementById('explore_tribes') + }, + methods: { + autoCompleteDestroyed: function() { + this.filters = []; + }, + autoCompleteChange: function(filters) { + this.filters = filters; + }, + outputToCSV: function() { + + let output = []; + + $.map(this.items, function(obj, key) { + output.push(obj._source.profile_id); + }); + + let url = `/api/v0.1/users_csv/?${$.param({profile_ids: output})}`; + + fetch(url) + .then(resp => resp.blob()) + .then(blob => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + + a.style.display = 'none'; + a.href = url; + a.download = `users_csv-${Date.now()}.csv`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + _alert('File download complete'); + }) + .catch((err) => { + console.log(err); + _alert('There was an issue downloading your file'); + }); + }, + + fetchMappings: function() { + let vm = this; + + $.when(vm.header.client.indices.getMapping()) + .then(response => { + vm.esColumns = response[vm.header.index]['mappings'][vm.header.type]['properties']; + vm.filterLoaded = true; + }); + } + }, + mounted() { + this.fetchMappings(); + this.fetch(this); + this.$watch('params', function(newVal, oldVal) { + this.searchUsers(); + }, { + deep: true + }); + }, + created() { + this.setHost(document.contxt.search_url ? document.contxt.search_url : 'https://elastic.gitcoin.co'); // TODO: set to proper env variable + this.setIndex('haystack'); + this.setType('modelresult'); + // this.extractURLFilters(); + }, + beforeMount() { + window.addEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }, false); + }, + beforeDestroy() { + window.removeEventListener('scroll', () => { + this.bottom = this.bottomVisible(); + }); + } + }); +} diff --git a/app/assets/v2/js/vue-components.js b/app/assets/v2/js/vue-components.js index 5f2a01be771..157b265b17d 100644 --- a/app/assets/v2/js/vue-components.js +++ b/app/assets/v2/js/vue-components.js @@ -78,12 +78,15 @@ Vue.component('modal', { Vue.component('select2', { - props: [ 'options', 'value' ], + props: [ 'options', 'value', 'placeholder', 'inputlength' ], template: '#select2-template', mounted: function() { let vm = this; - $(this.$el).select2({data: this.options}) + $(this.$el).select2({ + data: this.options, + placeHolder: this.placeholder !== null ? this.placeholder : 'filter here', + minimumInputLength: this.inputlength !== null ? this.inputlength : 1}) .val(this.value) .trigger('change') .on('change', function() { diff --git a/app/dashboard/models.py b/app/dashboard/models.py index 73ee2b4002b..6ac80f9d127 100644 --- a/app/dashboard/models.py +++ b/app/dashboard/models.py @@ -2621,6 +2621,7 @@ def get_queryset(self): return ProfileQuerySet(self.model, using=self._db).slim() + class Repo(SuperModel): name = models.CharField(max_length=255) @@ -4186,6 +4187,9 @@ def to_representation(instance): } + def to_es(self): + return json.dumps(self.to_dict()) + def to_dict(self): """Get the dictionary representation with additional data. @@ -4346,6 +4350,7 @@ def to_dict(self): return context + @property def reassemble_profile_dict(self): params = self.as_dict @@ -4450,6 +4455,85 @@ def post_logout(sender, request, user, **kwargs): from dashboard.utils import create_user_action create_user_action(user, 'Logout', request) +class UserDirectoryQuerySet(models.QuerySet): + """Define the Profile QuerySet to be used as the objects manager.""" + +class UserDirectoryManager(models.Manager): + def get_queryset(self): + return UserDirectoryQuerySet(self.model, using=self._db) + +class UserDirectory(models.Model): + profile_id = models.CharField(max_length=255, primary_key=True) + join_date = models.CharField(max_length=255) + github_created_at = models.CharField(max_length=255) + first_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + email = models.EmailField() + handle = models.CharField(max_length=255) + sms_verification = models.BooleanField() + persona = models.CharField(max_length=255) + rank_coder = models.IntegerField() + num_hacks_joined = models.IntegerField() + which_hacks_joined = ArrayField(base_field=models.IntegerField()) + hack_work_starts = models.IntegerField() + hack_work_submits = models.IntegerField() + hack_work_start_orgs = models.IntegerField() + hack_work_submit_orgs = models.IntegerField() + bounty_work_starts = models.IntegerField() + bounty_work_submits = models.IntegerField() + hack_started_feature = models.IntegerField() + hack_started_code_review = models.IntegerField() + hack_started_security = models.IntegerField() + hack_started_design = models.IntegerField() + hack_started_documentation = models.IntegerField() + hack_started_bug = models.IntegerField() + hack_started_other = models.IntegerField() + hack_started_improvement = models.IntegerField() + started_feature = models.IntegerField() + started_code_review = models.IntegerField() + started_security = models.IntegerField() + started_design = models.IntegerField() + started_documentation = models.IntegerField() + started_bug = models.IntegerField() + started_other = models.IntegerField() + started_improvement = models.IntegerField() + submitted_feature = models.IntegerField() + submitted_code_review = models.IntegerField() + submitted_security = models.IntegerField() + submitted_design = models.IntegerField() + submitted_documentation = models.IntegerField() + submitted_bug = models.IntegerField() + submitted_other = models.IntegerField() + submitted_improvement = models.IntegerField() + bounty_earnings = models.IntegerField() + bounty_work_start_orgs = models.IntegerField() + bounty_work_submit_orgs = models.IntegerField() + kudos_sends = models.IntegerField() + kudos_receives = models.IntegerField() + hack_winner_kudos_received = models.IntegerField() + grants_opened = models.IntegerField() + grant_contributed = models.IntegerField() + grant_contributions = models.IntegerField() + grant_contribution_amount = models.IntegerField() + num_actions = models.IntegerField() + action_points = models.FloatField() + avg_points_per_action = models.FloatField() + last_action_on = models.IntegerField() + keywords = ArrayField(base_field=models.CharField(max_length=255)) + activity_level = models.CharField(max_length=255) + reliability = models.CharField(max_length=255) + average_rating = models.IntegerField() + longest_streak = models.IntegerField() + earnings_count = models.IntegerField() + follower_count = models.IntegerField() + following_count = models.IntegerField() + num_repeated_relationships = models.IntegerField() + verification_status = models.IntegerField() + + objects = UserDirectoryManager() + + class Meta: + managed = False class ProfileSerializer(serializers.BaseSerializer): """Handle serializing the Profile object.""" diff --git a/app/dashboard/router.py b/app/dashboard/router.py index 33669c904aa..625650d1222 100644 --- a/app/dashboard/router.py +++ b/app/dashboard/router.py @@ -33,7 +33,7 @@ from .models import ( Activity, Bounty, BountyFulfillment, BountyInvites, HackathonEvent, HackathonProject, Interest, Profile, - ProfileSerializer, SearchHistory, TribeMember, + ProfileSerializer, SearchHistory, TribeMember, UserDirectory ) from .tasks import increment_view_count @@ -219,6 +219,16 @@ class Meta: class HackathonProjectsPagination(PageNumberPagination): page_size = 10 +class UserDirectorySerializer(serializers.ModelSerializer): + + class Meta: + model = UserDirectory + fields = '__all__' + depth = 1 + +class UserDirectoryPagination(PageNumberPagination): + page_size = 20 + class HackathonProjectsViewSet(viewsets.ModelViewSet): queryset = HackathonProject.objects.prefetch_related('bounty', 'profiles').all().order_by('id') diff --git a/app/dashboard/search_indexes.py b/app/dashboard/search_indexes.py new file mode 100644 index 00000000000..96789275c49 --- /dev/null +++ b/app/dashboard/search_indexes.py @@ -0,0 +1,85 @@ +from haystack import indexes + +from .models import UserDirectory + +class UserDirectoryIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + profile_id = indexes.IntegerField(null=True,model_attr='profile_id') + join_date = indexes.CharField(null=True,model_attr='join_date') + github_created_at = indexes.CharField(null=True,model_attr='github_created_at') + first_name = indexes.CharField(null=True,model_attr='first_name') + last_name = indexes.CharField(null=True,model_attr='last_name') + handle = indexes.CharField(null=True,model_attr='handle') + sms_verification = indexes.BooleanField(null=True,model_attr='sms_verification',faceted=True) + persona = indexes.CharField(null=True,model_attr='persona',faceted=True) + rank_coder = indexes.IntegerField(null=True,model_attr='rank_coder',faceted=True) + num_hacks_joined = indexes.IntegerField(null=True,model_attr='num_hacks_joined',faceted=True) + which_hacks_joined = indexes.MultiValueField(null=True,model_attr='which_hacks_joined',faceted=True) + hack_work_starts = indexes.IntegerField(null=True,model_attr='hack_work_starts',faceted=True) + hack_work_submits = indexes.IntegerField(null=True,model_attr='hack_work_submits',faceted=True) + hack_work_start_orgs = indexes.IntegerField(null=True,model_attr='hack_work_start_orgs',faceted=True) + hack_work_submit_orgs = indexes.IntegerField(null=True,model_attr='hack_work_submit_orgs',faceted=True) + bounty_work_starts = indexes.IntegerField(null=True,model_attr='bounty_work_starts',faceted=True) + bounty_work_submits = indexes.IntegerField(null=True,model_attr='bounty_work_submits',faceted=True) + hack_started_feature = indexes.IntegerField(null=True,model_attr='hack_started_feature',faceted=True) + hack_started_code_review = indexes.IntegerField(null=True,model_attr='hack_started_code_review',faceted=True) + hack_started_security = indexes.IntegerField(null=True,model_attr='hack_started_security',faceted=True) + hack_started_design = indexes.IntegerField(null=True,model_attr='hack_started_design',faceted=True) + hack_started_documentation = indexes.IntegerField(null=True,model_attr='hack_started_documentation',faceted=True) + hack_started_bug = indexes.IntegerField(null=True,model_attr='hack_started_bug',faceted=True) + hack_started_other = indexes.IntegerField(null=True,model_attr='hack_started_other',faceted=True) + hack_started_improvement = indexes.IntegerField(null=True,model_attr='hack_started_improvement',faceted=True) + started_feature = indexes.IntegerField(null=True,model_attr='started_feature',faceted=True) + started_code_review = indexes.IntegerField(null=True,model_attr='started_code_review',faceted=True) + started_security = indexes.IntegerField(null=True,model_attr='started_security',faceted=True) + started_design = indexes.IntegerField(null=True,model_attr='started_design',faceted=True) + started_documentation = indexes.IntegerField(null=True,model_attr='started_documentation',faceted=True) + started_bug = indexes.IntegerField(null=True,model_attr='started_bug',faceted=True) + started_other = indexes.IntegerField(null=True,model_attr='started_other',faceted=True) + started_improvement = indexes.IntegerField(null=True,model_attr='started_improvement',faceted=True) + submitted_feature = indexes.IntegerField(null=True,model_attr='submitted_feature',faceted=True) + submitted_code_review = indexes.IntegerField(null=True,model_attr='submitted_code_review',faceted=True) + submitted_security = indexes.IntegerField(null=True,model_attr='submitted_security',faceted=True) + submitted_design = indexes.IntegerField(null=True,model_attr='submitted_design',faceted=True) + submitted_documentation = indexes.IntegerField(null=True,model_attr='submitted_documentation',faceted=True) + submitted_bug = indexes.IntegerField(null=True,model_attr='submitted_bug',faceted=True) + submitted_other = indexes.IntegerField(null=True,model_attr='submitted_other',faceted=True) + submitted_improvement = indexes.IntegerField(null=True,model_attr='submitted_improvement',faceted=True) + bounty_earnings = indexes.IntegerField(null=True,model_attr='bounty_earnings',faceted=True) + bounty_work_start_orgs = indexes.IntegerField(null=True,model_attr='bounty_work_start_orgs',faceted=True) + bounty_work_submit_orgs = indexes.IntegerField(null=True,model_attr='bounty_work_submit_orgs',faceted=True) + kudos_sends = indexes.IntegerField(null=True,model_attr='kudos_sends',faceted=True) + kudos_receives = indexes.IntegerField(null=True,model_attr='kudos_receives',faceted=True) + hack_winner_kudos_received = indexes.IntegerField(null=True,model_attr='hack_winner_kudos_received',faceted=True) + grants_opened = indexes.IntegerField(null=True,model_attr='grants_opened',faceted=True) + grant_contributed = indexes.IntegerField(null=True,model_attr='grant_contributed',faceted=True) + grant_contributions = indexes.IntegerField(null=True,model_attr='grant_contributions',faceted=True) + grant_contribution_amount = indexes.IntegerField(null=True,model_attr='grant_contribution_amount',faceted=True) + num_actions = indexes.IntegerField(null=True,model_attr='num_actions',faceted=True) + action_points = indexes.IntegerField(null=True,model_attr='action_points',faceted=True) + avg_points_per_action = indexes.IntegerField(null=True,model_attr='avg_points_per_action',faceted=True) + last_action_on = indexes.CharField(null=True,model_attr='last_action_on') + keywords = indexes.MultiValueField(null=True,model_attr='keywords',faceted=True) + activity_level = indexes.CharField(null=True,model_attr='activity_level',faceted=True) + reliability = indexes.CharField(null=True,model_attr='reliability',faceted=True) + average_rating = indexes.IntegerField(null=True,model_attr='average_rating',faceted=True) + longest_streak = indexes.IntegerField(null=True,model_attr='longest_streak',faceted=True) + earnings_count = indexes.IntegerField(null=True,model_attr='earnings_count',faceted=True) + follower_count = indexes.IntegerField(null=True,model_attr='follower_count',faceted=True) + following_count = indexes.IntegerField(null=True,model_attr='following_count',faceted=True) + num_repeated_relationships = indexes.IntegerField(null=True,model_attr='num_repeated_relationships',faceted=True) + verification_status = indexes.CharField(null=True,model_attr='verification_status',faceted=True) + + # We add this for autocomplete. + handle_auto = indexes.EdgeNgramField(model_attr='handle') + keywords_auto = indexes.EdgeNgramField(model_attr='keywords') + first_name_auto = indexes.EdgeNgramField(null=True,model_attr='first_name') + last_name_auto = indexes.EdgeNgramField(null=True,model_attr='last_name') + persona_auto = indexes.EdgeNgramField(null=True,model_attr='persona') + + def get_model(self): + return UserDirectory + + def index_queryset(self, using=None): + """Used when the entire index for model is updated.""" + return self.get_model().objects.all() diff --git a/app/dashboard/templates/dashboard/users-elastic.html b/app/dashboard/templates/dashboard/users-elastic.html new file mode 100644 index 00000000000..7880630576d --- /dev/null +++ b/app/dashboard/templates/dashboard/users-elastic.html @@ -0,0 +1,287 @@ +{% comment %} + Copyright (C) 2020 Gitcoin Core + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +{% endcomment %} +{% load i18n static email_obfuscator add_url_schema avatar_tags %} + + + + + {% include 'shared/head.html' %} + {% include 'shared/cards.html' %} + + + + + + + + {% include 'shared/tag_manager_2.html' %} +
+ {% include 'shared/top_nav.html' with class='d-md-flex' %} + {% include 'home/nav.html' %} +
+
+
+ +
+
+ + + +
+ +
+ + + + +
+ nothing found +

No users found

+ +
+
+
+
+ +
+ + + Fund Bounty + +
+ + + View profile + +
+ + ${user._source.handle} Avatar + +
+ [[ user._source.name || user._source.handle ]] + + verify badget + +
+ + @[[ user._source.handle ]] + +
+ + ([[ user._source.avg_rating.total_rating ]] ratings) +
+ + Worked with you + +
+
+ + + + + + +
+
+
+
+
+
+
+ + + + {% csrf_token %} + {% include 'shared/analytics.html' %} + {% include 'shared/footer_scripts.html' %} + {% include 'shared/footer.html' %} + + + + + + + + + diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 8c65faccce0..949e5d5c0fa 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -111,7 +111,7 @@ CoinRedemptionRequest, Coupon, Earning, FeedbackEntry, HackathonEvent, HackathonProject, HackathonRegistration, HackathonSponsor, HackathonWorkshop, Interest, LabsResearch, Option, Poll, PortfolioItem, Profile, ProfileSerializer, ProfileVerification, ProfileView, Question, SearchHistory, Sponsor, Subscription, Tool, ToolVote, - TribeMember, UserAction, UserVerificationModel, + TribeMember, UserAction, UserDirectory, UserVerificationModel ) from .notifications import ( maybe_market_tip_to_email, maybe_market_tip_to_github, maybe_market_tip_to_slack, maybe_market_to_email, @@ -869,6 +869,28 @@ def users_directory(request): return TemplateResponse(request, 'dashboard/users.html', params) +@staff_member_required +def users_directory_elastic(request): + """Handle displaying users directory page.""" + from retail.utils import programming_languages, programming_languages_full + + keywords = programming_languages + programming_languages_full + + params = { + 'is_staff': request.user.is_staff, + 'avatar_url': request.build_absolute_uri(static('v2/images/twitter_cards/tw_cards-07.png')) , + 'active': 'users', + 'title': 'Users', + 'meta_title': "", + 'meta_description': "", + 'keywords': keywords + } + + if request.path == '/tribes/explore': + params['explore'] = 'explore_tribes' + + return TemplateResponse(request, 'dashboard/users-elastic.html', params) + def users_fetch_filters(profile_list, skills, bounties_completed, leaderboard_rank, rating, organisation, hackathon_id = ""): if not settings.DEBUG: @@ -960,6 +982,34 @@ def set_project_notes(request): return JsonResponse({}) +@require_GET +def users_autocomplete(request): + max_items = 5 + q = request.GET.get('q') + if q: + from haystack.query import SQ, SearchQuerySet + sqs = SearchQuerySet().autocomplete((SQ(first_name_auto=q) | SQ(last_name_auto=q) | SQ(handle_auto=q))) + results = [str(result.object) for result in sqs[:max_items]] + else: + results = [] + + return JsonResponse({ + 'results': results + }) + + +@require_GET +def output_users_to_csv(request): + + if request.user.is_authenticated and not request.user.is_staff: + return Http404() + + profile_ids = request.GET.getlist('profile_ids[]') + + user_query = UserDirectory.objects.filter(profile_id__in=profile_ids) + from djqscsv import render_to_csv_response + return render_to_csv_response(user_query) + @require_GET def users_fetch(request): """Handle displaying users.""" @@ -1167,6 +1217,7 @@ def previous_worked(): return JsonResponse(params, status=200, safe=False) + @require_POST def bounty_mentor(request): diff --git a/app/retail/templates/shared/footer_scripts.html b/app/retail/templates/shared/footer_scripts.html index 574ec1830eb..1967560f128 100644 --- a/app/retail/templates/shared/footer_scripts.html +++ b/app/retail/templates/shared/footer_scripts.html @@ -31,6 +31,8 @@ {% else %} + + {% endif %} {% include 'shared/sentry.html' %} diff --git a/docker-compose.yml b/docker-compose.yml index 88a95e045ab..284b25e2f08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,8 @@ services: ports: - "8065:8065" volumes: - - ./chatconfig:/mattermost/config/ + - ./chatdata/config:/mattermost/config/ + - ./chatdata/root.html:/mattermost/client/root.html depends_on: - db deploy: @@ -32,6 +33,15 @@ services: reservations: memory: 128M + elasticsearch: + image: launcher.gcr.io/google/elasticsearch2 + ports: + - "9200:9200" + - "9300:9300" + volumes: + - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + + ipfs: image: ipfs/go-ipfs:release restart: unless-stopped diff --git a/elasticsearch.yml b/elasticsearch.yml new file mode 100644 index 00000000000..06a6c9eceba --- /dev/null +++ b/elasticsearch.yml @@ -0,0 +1,3 @@ +network.host: 0.0.0.0 +http.cors.enabled: true +http.cors.allow-origin: "*" diff --git a/requirements/base.txt b/requirements/base.txt index d1d44884804..1e09346782b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,6 +7,7 @@ django-celery-beat==1.1.1 django==2.2.4 django-cors-headers==2.4.0 django-filter==2.0.0 +django-haystack django-ratelimit==1.1.0 djangorestframework==3.9.1 gitterpy @@ -26,6 +27,7 @@ python-twitter==3.2 sendgrid==5.6.0 slackclient==2.0.0 eth-tester +elasticsearch>=2.0.0,<3.0.0 websockets web3==4.5.0 eth-abi==1.1.1 @@ -93,7 +95,7 @@ redis==3.3.11 pandas wiki django-bulk-update -elasticsearch pdfrw django-admin-sortable2==0.7.6 twilio +django-queryset-csv diff --git a/scripts/crontab b/scripts/crontab index df25ffd5218..b829707ca39 100644 --- a/scripts/crontab +++ b/scripts/crontab @@ -103,6 +103,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/us ## INFRASTRUCTURE */15 * * * * cd gitcoin/coin; bash scripts/run_management_command.bash warm_cache >> /var/log/gitcoin/warm_cache.log 2>&1 +*/15 * * * * cd gitcoin/coin; bash scripts/run_management_command.bash update_index >> /var/log/gitcoin/elastic_search.log 2>&1 * * * * * date >> /var/log/gitcoin/running_procs.log; ps -aux | grep python3 >> /var/log/gitcoin/running_procs.log 2>&1 1 1 * * * rm -f /var/log/gitcoin/running_procs.log