From 0e6a00ac1124546b46fa04af47565302c60bef60 Mon Sep 17 00:00:00 2001 From: Jenny Fung Date: Fri, 18 Jan 2019 14:46:33 -0500 Subject: [PATCH 1/7] Export all survey tables to CSV Saves separate CSVs to the 'exports' folder Refs #384 --- .gitignore | 1 + src/icp/apps/beekeepers/tasks.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/.gitignore b/.gitignore index 44140a994..c5d14e877 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Script produced interim data src/icp/pollinator/src/pollinator/reclass/*.json src/icp/pollinator/src/pollinator/reclass/*.csv +src/icp/apps/beekeepers/exports/* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/src/icp/apps/beekeepers/tasks.py b/src/icp/apps/beekeepers/tasks.py index 6973a0d5f..eb6328da4 100644 --- a/src/icp/apps/beekeepers/tasks.py +++ b/src/icp/apps/beekeepers/tasks.py @@ -5,6 +5,14 @@ import boto3 import rasterio from pyproj import Proj +import psycopg2 + + +PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) +HOST = os.environ.get('ICP_DB_HOST') +DB_USER = os.environ.get('ICP_DB_USER') +DB_PW = os.environ.get('ICP_DB_PASSWORD') +DB_NAME = os.environ.get('ICP_DB_NAME') def sample_at_point(geom, raster_path): @@ -85,3 +93,47 @@ def __enter__(self): def __exit__(self, *args): """Context manager close""" self.remove_env_vars() + + +def export_survey_tables(): + """Export all types of beekeepers surveys to CSVs.""" + + db_options = "dbname={} user={} host={} password={}".format( + DB_NAME, DB_USER, HOST, DB_PW + ) + connection = psycopg2.connect(db_options) + cur = connection.cursor() + + tables = dict( + novembersurvey=None, + aprilsurvey=None, + monthlysurvey=None, + usersurvey=""" + SELECT auth_user.username AS username, auth_user.email AS email, + beekeepers_usersurvey.* + FROM beekeepers_usersurvey + INNER JOIN auth_user ON beekeepers_usersurvey.user_id=auth_user.id + """, + survey=""" + SELECT beekeepers_survey.*, beekeepers_apiary.lat AS lat, + beekeepers_apiary.lng AS lng + FROM beekeepers_survey + INNER JOIN beekeepers_apiary + ON beekeepers_survey.apiary_id=beekeepers_apiary.id + """, + ) + # collect output CSVs to dedicated folder + dir = os.path.join(PROJECT_ROOT, 'exports') + if not os.path.exists(dir): + os.mkdir(dir) + + for table, query in tables.iteritems(): + if query is None: + query = "SELECT * FROM beekeepers_{}".format(table) + output_query = "COPY ({0}) TO STDOUT WITH CSV HEADER".format(query) + + filename = "{}/{}.csv".format(dir, table) + with open(filename, 'w') as f: + cur.copy_expert(output_query, f) + + connection.close() From 527f50df109ad470fd8e03731e35e1fac1e769b7 Mon Sep 17 00:00:00 2001 From: Jenny Fung Date: Tue, 22 Jan 2019 11:44:57 -0500 Subject: [PATCH 2/7] Return zip of CSVs to user The CSVs are created in-memory and returned to the user so there are no artifacts left on the server. We don't expect the database to get large enough to warrant memory issues. Refs #384 --- src/icp/apps/beekeepers/tasks.py | 52 ------------------------ src/icp/apps/beekeepers/views.py | 70 +++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/src/icp/apps/beekeepers/tasks.py b/src/icp/apps/beekeepers/tasks.py index eb6328da4..6973a0d5f 100644 --- a/src/icp/apps/beekeepers/tasks.py +++ b/src/icp/apps/beekeepers/tasks.py @@ -5,14 +5,6 @@ import boto3 import rasterio from pyproj import Proj -import psycopg2 - - -PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -HOST = os.environ.get('ICP_DB_HOST') -DB_USER = os.environ.get('ICP_DB_USER') -DB_PW = os.environ.get('ICP_DB_PASSWORD') -DB_NAME = os.environ.get('ICP_DB_NAME') def sample_at_point(geom, raster_path): @@ -93,47 +85,3 @@ def __enter__(self): def __exit__(self, *args): """Context manager close""" self.remove_env_vars() - - -def export_survey_tables(): - """Export all types of beekeepers surveys to CSVs.""" - - db_options = "dbname={} user={} host={} password={}".format( - DB_NAME, DB_USER, HOST, DB_PW - ) - connection = psycopg2.connect(db_options) - cur = connection.cursor() - - tables = dict( - novembersurvey=None, - aprilsurvey=None, - monthlysurvey=None, - usersurvey=""" - SELECT auth_user.username AS username, auth_user.email AS email, - beekeepers_usersurvey.* - FROM beekeepers_usersurvey - INNER JOIN auth_user ON beekeepers_usersurvey.user_id=auth_user.id - """, - survey=""" - SELECT beekeepers_survey.*, beekeepers_apiary.lat AS lat, - beekeepers_apiary.lng AS lng - FROM beekeepers_survey - INNER JOIN beekeepers_apiary - ON beekeepers_survey.apiary_id=beekeepers_apiary.id - """, - ) - # collect output CSVs to dedicated folder - dir = os.path.join(PROJECT_ROOT, 'exports') - if not os.path.exists(dir): - os.mkdir(dir) - - for table, query in tables.iteritems(): - if query is None: - query = "SELECT * FROM beekeepers_{}".format(table) - output_query = "COPY ({0}) TO STDOUT WITH CSV HEADER".format(query) - - filename = "{}/{}.csv".format(dir, table) - with open(filename, 'w') as f: - cur.copy_expert(output_query, f) - - connection.close() diff --git a/src/icp/apps/beekeepers/views.py b/src/icp/apps/beekeepers/views.py index 02e3c674c..8319dfcb5 100644 --- a/src/icp/apps/beekeepers/views.py +++ b/src/icp/apps/beekeepers/views.py @@ -2,14 +2,18 @@ from __future__ import division import os +import psycopg2 +from cStringIO import StringIO +from zipfile import ZipFile +from tempfile import SpooledTemporaryFile -from django.http import Http404 +from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.utils.timezone import now from rest_framework import decorators, viewsets from rest_framework.response import Response -from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.permissions import AllowAny, IsAuthenticated, IsAdminUser from rest_framework.status import HTTP_204_NO_CONTENT from models import Apiary, Survey, UserSurvey @@ -22,6 +26,68 @@ DATA_BUCKET = os.environ['AWS_BEEKEEPERS_DATA_BUCKET'] +PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) +HOST = os.environ.get('ICP_DB_HOST') +DB_USER = os.environ.get('ICP_DB_USER') +DB_PW = os.environ.get('ICP_DB_PASSWORD') +DB_NAME = os.environ.get('ICP_DB_NAME') + + +@decorators.api_view(['GET']) +@decorators.permission_classes((IsAdminUser, )) +def export_survey_tables(request): + """Export a zip file of CSVs of all types of beekeepers surveys.""" + + # prepare queries to the database + db_options = "dbname={} user={} host={} password={}".format( + DB_NAME, DB_USER, HOST, DB_PW + ) + connection = psycopg2.connect(db_options) + cur = connection.cursor() + tables = dict( + novembersurvey=None, + aprilsurvey=None, + monthlysurvey=None, + usersurvey=""" + SELECT auth_user.username AS username, auth_user.email AS email, + beekeepers_usersurvey.* + FROM beekeepers_usersurvey + INNER JOIN auth_user ON beekeepers_usersurvey.user_id=auth_user.id + """, + survey=""" + SELECT beekeepers_survey.*, beekeepers_apiary.lat AS lat, + beekeepers_apiary.lng AS lng + FROM beekeepers_survey + INNER JOIN beekeepers_apiary + ON beekeepers_survey.apiary_id=beekeepers_apiary.id + """, + ) + + # the zipped CSVs are written in memory + date_stamp = now().strftime("%Y-%m-%d_%H-%M-%S") + zip_dir = 'beekeepers_exports_%s' % date_stamp + stream = StringIO() + + with ZipFile(stream, 'w') as zf: + for table, query in tables.iteritems(): + if query is None: + query = "SELECT * FROM beekeepers_{}".format(table) + + filename = "{}/{}_{}.csv".format(zip_dir, table, date_stamp) + full_query = "COPY ({0}) TO STDOUT WITH CSV HEADER".format(query) + tempfile = SpooledTemporaryFile() + cur.copy_expert(full_query, tempfile) + tempfile.seek(0) + zf.writestr(filename, tempfile.read()) + zf.close() + + resp = HttpResponse( + stream.getvalue(), + content_type="application/x-zip-compressed" + ) + resp['Content-Disposition'] = 'attachment; filename=%s.zip' % zip_dir + connection.close() + return resp @decorators.api_view(['POST']) From bc000dc02763951d292b83eb5aa18920b9c017f0 Mon Sep 17 00:00:00 2001 From: Jenny Fung Date: Tue, 22 Jan 2019 13:34:35 -0500 Subject: [PATCH 3/7] Download survey zip at /beekeepers/export The task suggested adding this functionality to Django's admin interface. The admin view is however not easily customizable. Downloading the zip file from an endpoint was more possible and straightforward and is pretty flexible. Refs #385 --- .gitignore | 1 - src/icp/apps/beekeepers/urls.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5d14e877..44140a994 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Script produced interim data src/icp/pollinator/src/pollinator/reclass/*.json src/icp/pollinator/src/pollinator/reclass/*.csv -src/icp/apps/beekeepers/exports/* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/src/icp/apps/beekeepers/urls.py b/src/icp/apps/beekeepers/urls.py index ea0493f32..41ed6be98 100644 --- a/src/icp/apps/beekeepers/urls.py +++ b/src/icp/apps/beekeepers/urls.py @@ -16,6 +16,7 @@ urlpatterns = patterns( '', + url(r'export/$', views.export_survey_tables, name='export'), url(r'fetch/$', views.fetch_data, name='fetch_data'), url(r'^apiary/(?P[0-9]+)/survey/$', views.create_survey, name='survey-create'), From 0fa7c4c5806b936c7444ec7484166660cdc08bbe Mon Sep 17 00:00:00 2001 From: Jenny Fung Date: Wed, 23 Jan 2019 10:15:47 -0500 Subject: [PATCH 4/7] Clean up --- .../beekeepers/js/src/components/Header.jsx | 27 +++++++++++++++-- src/icp/apps/beekeepers/views.py | 30 +++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/icp/apps/beekeepers/js/src/components/Header.jsx b/src/icp/apps/beekeepers/js/src/components/Header.jsx index 4e1e639f4..f429a1fd1 100644 --- a/src/icp/apps/beekeepers/js/src/components/Header.jsx +++ b/src/icp/apps/beekeepers/js/src/components/Header.jsx @@ -1,11 +1,23 @@ import React from 'react'; import { connect } from 'react-redux'; import { NavLink, withRouter } from 'react-router-dom'; -import { func, string } from 'prop-types'; +import { func, string, bool } from 'prop-types'; import { openParticipateModal, openLoginModal, logout } from '../actions'; -const Header = ({ dispatch, username }) => { +const Header = ({ dispatch, username, isStaff }) => { + const exportDataButton = isStaff + ? ( +
  • + + Export Survey Data + +
    • + ) : null; const authButtons = username ? (
  • @@ -16,7 +28,14 @@ const Header = ({ dispatch, username }) => { {username} ▾ - {/* Hidden Log Out button for screen readers */} + {/* Hidden buttons for screen readers */} + + Export Survey Data +
      + {exportDataButton}
      • ) : null; + const exportDataButtonForScreenReader = isStaff + ? ( +
    • + + Export Survey Data + +
      • + ) : null; const authButtons = username ? (
    • @@ -33,13 +45,7 @@ const Header = ({ dispatch, username, isStaff }) => { ▾ {/* Hidden buttons for screen readers */} - - Export Survey Data - + {exportDataButtonForScreenReader} + ); + const authButtons = username ? (
    • @@ -46,23 +52,11 @@ const Header = ({ dispatch, username, isStaff }) => { {/* Hidden buttons for screen readers */} {exportDataButtonForScreenReader} - + {makeLogOutButton(true)}
        {exportDataButton}
      • - + {makeLogOutButton()}
      • @@ -89,6 +83,7 @@ const Header = ({ dispatch, username, isStaff }) => { ); + return (
      diff --git a/src/icp/apps/beekeepers/views.py b/src/icp/apps/beekeepers/views.py index 4708cc82c..ad24f6be8 100644 --- a/src/icp/apps/beekeepers/views.py +++ b/src/icp/apps/beekeepers/views.py @@ -81,7 +81,7 @@ def export_survey_tables(request): resp = HttpResponse( stream.getvalue(), - content_type='application/x-zip-compressed' + content_type='application/zip' ) resp['Content-Disposition'] = 'attachment; filename={}.zip'.format(zip_dir) connection.close()