Skip to content

Commit

Permalink
Merge pull request #351 from mehtadev17/elasticsearch_updates
Browse files Browse the repository at this point in the history 
Elasticsearch updates
  • Loading branch information
@sephcoster
sephcoster committed Aug 11, 2015
2 parents 419ace4 + 60f5e3e commit 7346c9a
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 52 deletions.
4 changes: 4 additions & 0 deletions mapusaurus/censusdata/tests/test_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ def test_race_summary(self):
self.assertEqual(resp['1122233400']['non_hisp_asian_only_perc'], .2)

def test_race_summary_csv(self):
resp = self.client.get(reverse('censusdata:race_summary_csv'))
self.assertEqual(resp.status_code, 404)

resp = self.client.get(reverse('censusdata:race_summary_csv'),
{'metro':'10000', 'lender':'91000000001', 'action_taken':'1,2,3,4'})
resp = resp.content
Expand All @@ -107,3 +110,4 @@ def test_race_summary_csv(self):




82 changes: 44 additions & 38 deletions mapusaurus/censusdata/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from django.utils.encoding import smart_str
from django.http import HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404
from django.http import Http404

from .models import Census2010RaceStats
from geo.views import get_censustract_geos
Expand Down Expand Up @@ -226,43 +227,48 @@ def race_summary_http(request):
return HttpResponse(json.dumps(race_summary_as_json(request)))

def race_summary_csv(request):
lar_data = loan_originations_as_json(request)
tracts_in_msa = get_censustract_geos(request)
queryset = Census2010RaceStats.objects.filter(geoid__in=tracts_in_msa)
file_name = 'HMDA-Census-Tract_2013_Lender%s_MSA%s.csv' % (request.GET.get('lender'), request.GET.get('metro'))
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=%s' % file_name
writer = csv.writer(response, csv.excel)
#response.write(u'\ufeff'.encode('utf8')) # BOM (optional...Excel needs it to open UTF-8 file properly)
writer.writerow([
smart_str(u"geoid"),
smart_str(u"Total Population"),
smart_str(u"Hispanic Percentage"),
smart_str(u"White Only Percentage"),
smart_str(u"Non Hispanic Black Only Percentage"),
smart_str(u"Non Hispanic Asian Only Percentage"),
smart_str(u"HMDA LAR Count"),
smart_str(u"Total Households"),
])
for obj in queryset:
geoid = "'%s'" % str(obj.geoid.geoid)
try:
lar_count = lar_data[obj.geoid.geoid]['volume']
except:
lar_count = 0
try:
num_households = lar_data[obj.geoid.geoid]['num_households']
except:
num_households = 0

institution_id = request.GET.get('lender')
metro = request.GET.get('metro')
if institution_id and metro:
lar_data = loan_originations_as_json(request)
tracts_in_msa = get_censustract_geos(request)
queryset = Census2010RaceStats.objects.filter(geoid__in=tracts_in_msa)
file_name = 'HMDA-Census-Tract_2013_Lender%s_MSA%s.csv' % (institution_id, metro)
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=%s' % file_name
writer = csv.writer(response, csv.excel)
writer.writerow([
smart_str(geoid),
smart_str(obj.total_pop),
smart_str(obj.hispanic_perc * 100),
smart_str(obj.non_hisp_white_only_perc * 100),
smart_str(obj.non_hisp_black_only_perc * 100),
smart_str(obj.non_hisp_asian_only_perc * 100),
smart_str(lar_count),
smart_str(num_households),
smart_str(u"geoid"),
smart_str(u"Total Population"),
smart_str(u"Hispanic Percentage"),
smart_str(u"White Only Percentage"),
smart_str(u"Non Hispanic Black Only Percentage"),
smart_str(u"Non Hispanic Asian Only Percentage"),
smart_str(u"HMDA LAR Count"),
smart_str(u"Total Households"),
])
return response
for obj in queryset:
geoid = "'%s'" % str(obj.geoid.geoid)
try:
lar_count = lar_data[obj.geoid.geoid]['volume']
except:
lar_count = 0
try:
num_households = lar_data[obj.geoid.geoid]['num_households']
except:
num_households = 0

writer.writerow([
smart_str(geoid),
smart_str(obj.total_pop),
smart_str(obj.hispanic_perc * 100),
smart_str(obj.non_hisp_white_only_perc * 100),
smart_str(obj.non_hisp_black_only_perc * 100),
smart_str(obj.non_hisp_asian_only_perc * 100),
smart_str(lar_count),
smart_str(num_households),
])
return response
else:
raise Http404("Invalid Institution or Metro")

6 changes: 0 additions & 6 deletions mapusaurus/respondents/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ def test_search_name(self, SQS):


self.assertTrue('Bank' in str(SQS.filter.call_args))
self.assertTrue('content' in str(SQS.filter.call_args))
self.assertTrue('Some Bank' in resp.content)
self.assertTrue('Bank & Loan' in resp.content)
self.assertRaises(ValueError, json.loads, resp.content)
Expand All @@ -182,7 +181,6 @@ def test_search_autocomplete(self, SQS):
self.client.get(reverse('respondents:search_results'),
{'q': 'Bank', 'auto': '1'})
self.assertTrue('Bank' in str(SQS.filter.call_args))
self.assertFalse('content' in str(SQS.filter.call_args))
self.assertTrue('text_auto' in str(SQS.filter.call_args))

@patch('respondents.views.SearchQuerySet')
Expand All @@ -198,14 +196,12 @@ def test_search_id(self, SQS):
{'q': '01234567'})

self.assertTrue('01234567' in str(SQS.filter.call_args))
self.assertTrue('content' in str(SQS.filter.call_args))
self.assertTrue('Some Bank' in resp.content)
self.assertRaises(ValueError, json.loads, resp.content)

resp = self.client.get(reverse('respondents:search_results'),
{'q': '012345-7899'})
self.assertTrue('012345-7899' in str(SQS.filter.call_args))
self.assertFalse('content' in str(SQS.filter.call_args))
self.assertTrue('lender_id' in str(SQS.filter.call_args))
self.assertTrue('Some Bank' in resp.content)
self.assertRaises(ValueError, json.loads, resp.content)
Expand All @@ -215,15 +211,13 @@ def test_search_id(self, SQS):
resp = self.client.get(reverse('respondents:search_results'),
{'q': q})
self.assertTrue('01123456799' in str(SQS.filter.call_args))
self.assertFalse('content' in str(SQS.filter.call_args))
self.assertTrue('lender_id' in str(SQS.filter.call_args))
self.assertTrue('Some Bank' in resp.content)
self.assertRaises(ValueError, json.loads, resp.content)

resp = self.client.get(reverse('respondents:search_results'),
{'q': 'Some Bank (0112345799)'})
self.assertTrue('0112345799' in str(SQS.filter.call_args))
self.assertTrue('content' in str(SQS.filter.call_args))
self.assertFalse('lender_id' in str(SQS.filter.call_args))
self.assertTrue('Some Bank' in resp.content)
self.assertRaises(ValueError, json.loads, resp.content)
Expand Down
17 changes: 9 additions & 8 deletions mapusaurus/respondents/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from rest_framework.response import Response
from django.http import HttpResponseBadRequest
from respondents.models import Institution, Branch
from django.utils.html import escape


def respondent(request, agency_id, respondent):
Expand Down Expand Up @@ -74,7 +75,7 @@ class Meta:

@api_view(['GET'])
def search_results(request):
query_str = request.GET.get('q', '').strip()
query_str = escape(request.GET.get('q', '')).strip()
lender_id = False
respondent_id = False
for regex in LENDER_REGEXES:
Expand All @@ -97,7 +98,7 @@ def search_results(request):
query = query.filter(lender_id=Exact(lender_id))
elif respondent_id:
query = query.filter(respondent_id=Exact(respondent_id))
elif query_str and request.GET.get('auto'):
elif query_str and escape(request.GET.get('auto')):
query = query.filter(text_auto=AutoQuery(query_str))
elif query_str:
query = query.filter(content=AutoQuery(query_str))
Expand All @@ -124,7 +125,7 @@ def search_results(request):
start_results = 0
end_results = num_results

sort = current_sort #request.GET.get('sort', 'relevance')
sort = current_sort

total_results = len(query)

Expand Down Expand Up @@ -170,11 +171,11 @@ def branch_locations_as_json(request):

def branch_locations(request):
"""This endpoint returns geocoded branch locations"""
lender = request.GET.get('lender')
northEastLat = request.GET.get('neLat')
northEastLon = request.GET.get('neLon')
southWestLat = request.GET.get('swLat')
southWestLon = request.GET.get('swLon')
lender = escape(request.GET.get('lender'))
northEastLat = escape(request.GET.get('neLat'))
northEastLon = escape(request.GET.get('neLon'))
southWestLat = escape(request.GET.get('swLat'))
southWestLon = escape(request.GET.get('swLon'))
try:
maxlat, minlon, minlat, maxlon = float(northEastLat), float(southWestLon), float(southWestLat), float(northEastLon)
except ValueError:
Expand Down

0 comments on commit 7346c9a

Please sign in to comment.