Skip to content

Commit

Permalink
Merge pull request #634 from gitcoinco/kevin/refactor_3_requests
Browse files Browse the repository at this point in the history 
Refactors 3 requests down to 1
  • Loading branch information
@owocki
owocki authored Mar 16, 2018
2 parents bfa7fd2 + cd5148c commit f3b4bc6
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 169 deletions.
4 changes: 1 addition & 3 deletions app/app/urls.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,8 @@

# sync methods
url(r'^sync/web3', dashboard.views.sync_web3, name='sync_web3'),
url(r'^sync/get_issue_title?', dashboard.helpers.title, name='helpers_title'),
url(r'^sync/get_issue_description?', dashboard.helpers.description, name='helpers_description'),
url(r'^sync/get_amount?', dashboard.helpers.amount, name='helpers_amount'),
url(r'^sync/get_issue_keywords?', dashboard.helpers.keywords, name='helpers_keywords'),
url(r'^sync/get_issue_details?', dashboard.helpers.issue_details, name='helpers_issue_details'),
url(r'^sync/search_save?', dashboard.views.save_search, name='save_search'),
# brochureware views
url(r'^about/?', retail.views.about, name='about'),
Expand Down
8 changes: 2 additions & 6 deletions app/assets/v2/js/pages/new_bounty.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,10 @@ $(document).ready(function() {
$('input[name=amount]').keyup(setUsdAmount);
$('input[name=amount]').blur(setUsdAmount);
$('select[name=deonomination]').change(setUsdAmount);
$('input[name=issueURL]').blur(retrieveTitle);
$('input[name=issueURL]').blur(retrieveKeywords);
$('input[name=issueURL]').blur(retrieveDescription);
$('input[name=issueURL]').blur(retrieveIssueDetails);

if ($('input[name=issueURL]').val() != '') {
retrieveTitle();
retrieveKeywords();
retrieveDescription();
retrieveIssueDetails();
}
$('input[name=issueURL]').focus();

Expand Down
78 changes: 22 additions & 56 deletions app/assets/v2/js/shared.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,34 +408,13 @@ var updateAmountUI = function(target_ele, usd_amount) {
target_ele.html('Approx: ' + usd_amount + ' USD');
};

var retrieveTitle = function() {
var retrieveIssueDetails = function() {
var ele = $('input[name=issueURL]');
var target_ele = $('input[name=title]');
var issue_url = ele.val();

if (typeof issue_url == 'undefined') {
return;
}
if (issue_url.length < 5 || issue_url.indexOf('github') == -1) {
return;
}
var request_url = '/sync/get_issue_title?url=' + encodeURIComponent(issue_url);

target_ele.addClass('loading');
$.get(request_url, function(result) {
result = sanitizeAPIResults(result);
target_ele.removeClass('loading');
if (result['title']) {
target_ele.val(result['title']);
}
}).fail(function() {
target_ele.removeClass('loading');
});
};

var retrieveDescription = function() {
var ele = $('input[name=issueURL]');
var target_ele = $('textarea[name=description]');
var target_eles = {
'title': $('input[name=title]'),
'keywords': $('input[name=keywords]'),
'description': $('textarea[name=description]')
};
var issue_url = ele.val();

if (typeof issue_url == 'undefined') {
Expand All @@ -444,44 +423,31 @@ var retrieveDescription = function() {
if (issue_url.length < 5 || issue_url.indexOf('github') == -1) {
return;
}
var request_url = '/sync/get_issue_description?url=' + encodeURIComponent(issue_url);
var request_url = '/sync/get_issue_details?url=' + encodeURIComponent(issue_url);

target_ele.addClass('loading');
$.get(request_url, function(result) {
result = sanitizeAPIResults(result);
target_ele.removeClass('loading');
if (result['description']) {
target_ele.val(result['description']);
}
}).fail(function() {
target_ele.removeClass('loading');
$.each(target_eles, function(i, ele) {
ele.addClass('loading');
});
};

var retrieveKeywords = function() {
var ele = $('input[name=issueURL]');
var target_ele = $('input[name=keywords]');
var issue_url = ele.val();

if (typeof issue_url == 'undefined') {
return;
}
if (issue_url.length < 5 || issue_url.indexOf('github') == -1) {
return;
}
var request_url = '/sync/get_issue_keywords?url=' + encodeURIComponent(issue_url);

target_ele.addClass('loading');
$.get(request_url, function(result) {
result = sanitizeAPIResults(result);
target_ele.removeClass('loading');
if (result['keywords']) {
var keywords = result['keywords'];

target_ele.val(keywords.join(', '));
target_eles['keywords'].val(keywords.join(', '));
}
if (result['description']) {
target_eles['description'].val(result['description']);
}
if (result['title']) {
target_eles['title'].val(result['title']);
}
$.each(target_eles, function(i, ele) {
ele.removeClass('loading');
});
}).fail(function() {
target_ele.removeClass('loading');
$.each(target_eles, function(i, ele) {
ele.removeClass('loading');
});
});
};

Expand Down
94 changes: 12 additions & 82 deletions app/dashboard/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,70 +75,16 @@ def amount(request):
raise Http404


# gets title of remote html doc (github issue)
@ratelimit(key='ip', rate='10/m', method=ratelimit.UNSAFE, block=True)
def title(request):
"""Determine the Github issue title of the specified Github issue URL.
Returns:
JsonResponse: A JSON response containing the Github issue title.
"""
response = {}

url = request.GET.get('url')
url_val = URLValidator()
try:
url_val(url)
except ValidationError:
response['message'] = 'invalid arguments'
return JsonResponse(response)

if url.lower()[:19] != 'https://github.com/':
response['message'] = 'invalid arguments'
return JsonResponse(response)

try:
html_response = requests.get(url)
except ValidationError:
response['message'] = 'could not pull back remote response'
return JsonResponse(response)

title = None
try:
soup = BeautifulSoup(html_response.text, 'html.parser')

eles = soup.findAll("span", {"class": "js-issue-title"})
if eles:
title = eles[0].text

if not title and soup.title:
title = soup.title.text

if not title:
for link in soup.find_all('h1'):
print(link.text)

except ValidationError:
response['message'] = 'could not parse html'
return JsonResponse(response)

try:
response['title'] = title.replace('\n', '').strip()
except Exception as e:
print(e)
response['message'] = 'could not find a title'

return JsonResponse(response)

@ratelimit(key='ip', rate='50/m', method=ratelimit.UNSAFE, block=True)
def issue_details(request):
"""Determine the Github issue keywords of the specified Github issue or PR URL.
# gets description of remote html doc (github issue)
@ratelimit(key='ip', rate='10/m', method=ratelimit.UNSAFE, block=True)
def description(request):
"""Determine the Github issue description of the specified Github issue URL.
Todo:
* Modify the view to only use the Github API (remove BeautifulSoup).
* Simplify the view logic.
Returns:
JsonResponse: A JSON response containing the Github issue description.
JsonResponse: A JSON response containing the Github issue or PR keywords.
"""
response = {}
Expand Down Expand Up @@ -170,27 +116,14 @@ def description(request):
return JsonResponse(response)

try:
body = api_response.json()['body']
except ValueError as e:
response['message'] = str(e)
except KeyError as e:
response = api_response.json()
body = response['body']
except (KeyError, ValueError) as e:
response['message'] = str(e)
else:
response['description'] = body.replace('\n', '').strip()
response['title'] = response['title']

return JsonResponse(response)


# gets keywords of remote issue (github issue)
@ratelimit(key='ip', rate='10/m', method=ratelimit.UNSAFE, block=True)
def keywords(request):
"""Determine the Github issue keywords of the specified Github issue or PR URL.
Returns:
JsonResponse: A JSON response containing the Github issue or PR keywords.
"""
response = {}
keywords = []

url = request.GET.get('url')
Expand All @@ -216,10 +149,7 @@ def keywords(request):
keywords.append(split_repo_url[-2])

html_response = requests.get(repo_url)
except ValidationError:
response['message'] = 'could not pull back remote response'
return JsonResponse(response)
except AttributeError:
except (AttributeError, ValidationError):
response['message'] = 'could not pull back remote response'
return JsonResponse(response)

Expand Down
32 changes: 10 additions & 22 deletions app/dashboard/tests/test_dashboard_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import json

from django.test import TestCase
from django.test.client import RequestFactory

import requests_mock
from dashboard.helpers import amount, description, keywords, normalize_url, title
from dashboard.helpers import amount, issue_details, normalize_url
from economy.models import ConversionRate


Expand All @@ -45,32 +47,18 @@ def test_amount(self):
request = self.factory.get('/sync/get_amount', params)
assert amount(request).content == b'{"eth": 5.0, "usdt": 10.0}'

def test_title(self):
"""Test the dashboard helper title method."""
sample_url = 'https://github.com/gitcoinco/web/issues/353'
params = {'url': sample_url, 'denomination': 'ETH'}
with requests_mock.Mocker() as m:
m.get(sample_url, text='<span class="js-issue-title refined-linkified-title">Increase Code Coverage by 4%</span>')
request = self.factory.get('/sync/get_issue_title', params)
assert title(request).content == b'{"title": "Increase Code Coverage by 4%"}'

def test_description(self):
def test_issue_details(self):
"""Test the dashboard helper description method."""
sample_url = 'https://github.com/gitcoinco/web/issues/353'
params = {'url': sample_url}
with requests_mock.Mocker() as m:
m.get('https://api.github.com/repos/gitcoinco/web/issues/353', text='{"body": "This bounty will be paid out to anyone who meaningfully increases the code coverage of the repository by 4%."}')
request = self.factory.get('/sync/get_issue_description', params)
assert description(request).content == b'{"description": "This bounty will be paid out to anyone who meaningfully increases the code coverage of the repository by 4%."}'

def test_keywords(self):
"""Test the dashboard helper keywords method."""
sample_url = 'https://github.com/gitcoinco/web/issues/353'
params = {'url': sample_url}
with requests_mock.Mocker() as m:
m.get('https://api.github.com/repos/gitcoinco/web/issues/353', text='{ "title":"Increase Code Coverage by 4%", "body": "This bounty will be paid out to anyone who meaningfully increases the code coverage of the repository by 4%."}')
m.get('https://github.com/gitcoinco/web', text='<span class="lang">hello</span><span class="lang">world</span>')
request = self.factory.get('/sync/get_issue_keywords', params)
assert keywords(request).content == b'{"keywords": ["web", "gitcoinco", "hello", "world"]}'
request = self.factory.get('/sync/get_issue_details', params)
response = json.loads(issue_details(request).content)
assert response['description'] == "This bounty will be paid out to anyone who meaningfully increases the code coverage of the repository by 4%."
assert response['keywords'] == ["web", "gitcoinco", "hello", "world"]
assert response['title'] == "Increase Code Coverage by 4%"

def test_normalize_url(self):
"""Test the dashboard helper normalize_url method."""
Expand Down

0 comments on commit f3b4bc6

Please sign in to comment.