From 6a00ea5c255ff945753b5b73638c34cd3579c18b Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 00:38:05 -0800 Subject: [PATCH 01/20] Add models for parsing our google sheets --- ballot/admin.py | 12 +++----- ballot_api/settings.py | 1 + ballot_api/urls.py | 5 ++- gsheets/__init__.py | 0 gsheets/admin.py | 8 +++++ gsheets/migrations/__init__.py | 0 gsheets/models.py | 56 ++++++++++++++++++++++++++++++++++ gsheets/tests.py | 3 ++ gsheets/views.py | 3 ++ 9 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 gsheets/__init__.py create mode 100644 gsheets/admin.py create mode 100644 gsheets/migrations/__init__.py create mode 100644 gsheets/models.py create mode 100644 gsheets/tests.py create mode 100644 gsheets/views.py diff --git a/ballot/admin.py b/ballot/admin.py index daa3a78..fb5b5c0 100644 --- a/ballot/admin.py +++ b/ballot/admin.py @@ -14,11 +14,7 @@ class PersonAdmin(admin.ModelAdmin): exclude = ('filer',) -class BallotApiAdminSite(admin.AdminSite): - site_header = 'Open Disclosure Ballot API Admin' - - -api_admin = BallotApiAdminSite(name='ballotapi-admin') -api_admin.register(Election) -api_admin.register(Person, PersonAdmin) -api_admin.register(State) +admin.site.site_header = 'Open Disclosure Ballot API Admin' +admin.site.register(Election) +admin.site.register(Person, PersonAdmin) +admin.site.register(State) diff --git a/ballot_api/settings.py b/ballot_api/settings.py index 779755d..00653f3 100644 --- a/ballot_api/settings.py +++ b/ballot_api/settings.py @@ -18,6 +18,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'ballot', + 'gsheets', 'rest_framework_swagger', ) diff --git a/ballot_api/urls.py b/ballot_api/urls.py index 3fa1967..40b7f54 100644 --- a/ballot_api/urls.py +++ b/ballot_api/urls.py @@ -1,11 +1,10 @@ from django.conf.urls import url, include +from django.contrib import admin from rest_framework import routers from rest_framework_swagger.views import get_swagger_view -from ballot.admin import api_admin from ballot.views import CandidateViewSet, ElectionViewSet - router = routers.DefaultRouter() router.register(r'candidates', CandidateViewSet) router.register(r'elections', ElectionViewSet) @@ -16,5 +15,5 @@ url(r'^api/', include(router.urls)), url(r'^$', schema_view), url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', api_admin.urls), + url(r'^admin/', include(admin.site.urls)), ] diff --git a/gsheets/__init__.py b/gsheets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gsheets/admin.py b/gsheets/admin.py new file mode 100644 index 0000000..fe6b360 --- /dev/null +++ b/gsheets/admin.py @@ -0,0 +1,8 @@ +from django.contrib import admin + +from .models import Candidate, Committee, Referendum, ReferendumMapping + +admin.site.register(Candidate) +admin.site.register(Referendum) +admin.site.register(ReferendumMapping) +admin.site.register(Committee) diff --git a/gsheets/migrations/__init__.py b/gsheets/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gsheets/models.py b/gsheets/models.py new file mode 100644 index 0000000..4ee8bea --- /dev/null +++ b/gsheets/models.py @@ -0,0 +1,56 @@ +from django.db import models + + +class Candidate(models.Model): + fppc = models.IntegerField(blank=True, null=True, unique=True) + committee_name = models.CharField(max_length=60) + candidate = models.CharField(max_length=30) + aliases = models.TextField( + blank=True, + help_text='Comma separated list of alternative spellings and names within the filings.') + office = models.CharField(blank=True, max_length=30, help_text='Office the candidate is running for.') + incumbent = models.BooleanField() + accepted_expenditure_ceiling = models.BooleanField() + website = models.URLField(blank=True, null=True) + twitter = models.URLField(blank=True, null=True) + party_affiliation = models.CharField(blank=True, max_length=15) + occupation = models.CharField(blank=True, max_length=30) + bio = models.TextField(blank=True) + photo = models.ImageField(blank=True, null=True) + votersedge = models.URLField(blank=True, null=True) + + +class Referendum(models.Model): + measure_number = models.CharField(max_length=5) + short_title = models.CharField(blank=True, max_length=30) + full_title = models.TextField(blank=True) + summary = models.TextField() + votersedge = models.URLField(blank=True, null=True) + + +class Committee(models.Model): + COMMITTEE_TYPES = [ + ('RCP', 'RCP'), + ('BMC', 'BMC'), + ] + + SUPPORT_OPPOSE = [ + ('S', 'Support'), + ('O', 'Oppose'), + ] + + filer_id = models.IntegerField() + filer_naml = models.CharField(blank=True, max_length=20) + committee_type = models.CharField(blank=True, max_length=3, choices=COMMITTEE_TYPES) + description = models.CharField(blank=True, max_length=20) + ballot_measure = models.CharField(blank=True, max_length=3) + support_or_oppose = models.CharField(blank=True, max_length=1, choices=SUPPORT_OPPOSE) + website = models.URLField(blank=True, null=True) + twitter = models.URLField(blank=True, null=True) + facebook = models.URLField(blank=True, null=True) + netfilelocalid = models.CharField(blank=True, max_length=15) + + +class ReferendumMapping(models.Model): + measure_name = models.CharField(max_length=30) + measure_number = models.CharField(max_length=3) diff --git a/gsheets/tests.py b/gsheets/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/gsheets/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/gsheets/views.py b/gsheets/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/gsheets/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From d5dae94d30bd90e3c343adc9877d7cc5233096fc Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 16:37:09 -0800 Subject: [PATCH 02/20] Import task for Candidates --- ballot_api/urls.py | 10 +- gsheets/management/__init__.py | 0 gsheets/management/commands/__init__.py | 0 gsheets/management/commands/gsheets_import.py | 94 +++++++++++++++++++ gsheets/models.py | 17 ++-- gsheets/parsers.py | 0 gsheets/serializers.py | 10 ++ gsheets/views.py | 12 ++- 8 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 gsheets/management/__init__.py create mode 100644 gsheets/management/commands/__init__.py create mode 100644 gsheets/management/commands/gsheets_import.py create mode 100644 gsheets/parsers.py create mode 100644 gsheets/serializers.py diff --git a/ballot_api/urls.py b/ballot_api/urls.py index 40b7f54..7f64a4b 100644 --- a/ballot_api/urls.py +++ b/ballot_api/urls.py @@ -3,11 +3,13 @@ from rest_framework import routers from rest_framework_swagger.views import get_swagger_view -from ballot.views import CandidateViewSet, ElectionViewSet +import ballot.views as ballot_views +import gsheets.views as gsheets_views -router = routers.DefaultRouter() -router.register(r'candidates', CandidateViewSet) -router.register(r'elections', ElectionViewSet) +router = routers.SimpleRouter() +#router.register(r'candidates', ballot_views.CandidateViewSet) +router.register(r'elections', ballot_views.ElectionViewSet) +router.register(r'candidates', gsheets_views.CandidateViewSet) schema_view = get_swagger_view(title='Open Disclosure Ballot API') diff --git a/gsheets/management/__init__.py b/gsheets/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gsheets/management/commands/__init__.py b/gsheets/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py new file mode 100644 index 0000000..886eb26 --- /dev/null +++ b/gsheets/management/commands/gsheets_import.py @@ -0,0 +1,94 @@ +import csv +import io +import logging +import re +import tempfile +import urllib.request + +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand + +from gsheets.models import Candidate +from gsheets.serializers import CandidateSerializer + + +CANDIDATES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=csv' # noqa +REFERENDUMS_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1693935349&single=true&output=csv' # noqa +REFERENDUM_NAME_TO_NUMBER_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=896561174&single=true&output=csv' # noqa +COMMITTEES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1995437960&single=true&output=csv' # noqa + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Import data from Google Sheets' + + def add_arguments(self, parser): + parser.add_argument( + '--force', + action='store_true', + default=False, + help="Import the entity even if it already exists.") + + def handle(self, *args, **options): + force = options['force'] + + with urllib.request.urlopen(CANDIDATES_SHEET) as request: + with tempfile.TemporaryFile() as csvfile: + csvfile.write(request.read()) + csvfile.seek(0) + reader = csv.DictReader(io.TextIOWrapper(csvfile, encoding='utf-8')) + + total = 0 + imported = 0 + import_errors = 0 + for row in reader: + total += 1 + name = row.get('Candidate', None) + if not name: + continue + + exists = True + try: + Candidate.objects.get(candidate=name) + except ObjectDoesNotExist: + exists = False + + if exists and not force: + continue + field_names = [field.name for field in Candidate._meta.get_fields()] + model = ((k.lower(), v) for k, v in row.items()) + model = ((re.sub(r'\s+', '_', k), v) for k, v in model) + model = ((re.sub(r'[^a-z_]', '', k), v) for k, v in model) + model = dict((k, v) for k, v in model if k in field_names) + + # Convert fppc + fppc = model.get('fppc', None) + if not fppc: + model['fppc'] = None + + # Parse booleans + accepted_expenditure_ceiling = model.get('accepted_expenditure_ceiling', False) + model['accepted_expenditure_ceiling'] = bool(accepted_expenditure_ceiling) + + # Convert twitter @handle to URL + twitter = model.get('twitter', None) + if twitter: + # Drop the first char (@) + model['twitter'] = 'https://twitter.com/%s' % twitter[1:] + + logger.debug(model) + serializer = CandidateSerializer(data=model) + if not serializer.is_valid(): + import_errors += 1 + logger.error('Candidate could not be parsed: name="%s" parse_errors=%s row=%s', + name, serializer.errors, model) + continue + + Candidate.objects.update_or_create(candidate=name, defaults=serializer.validated_data) + imported += 1 + logger.info('Imported candidate "%s"' % name) + + logger.info('Import candidates complete: total=%s imported=%s errors=%s', + total, imported, import_errors) diff --git a/gsheets/models.py b/gsheets/models.py index 4ee8bea..1f52b74 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -3,22 +3,23 @@ class Candidate(models.Model): fppc = models.IntegerField(blank=True, null=True, unique=True) - committee_name = models.CharField(max_length=60) + committee_name = models.CharField(blank=True, max_length=120) candidate = models.CharField(max_length=30) - aliases = models.TextField( - blank=True, - help_text='Comma separated list of alternative spellings and names within the filings.') office = models.CharField(blank=True, max_length=30, help_text='Office the candidate is running for.') - incumbent = models.BooleanField() - accepted_expenditure_ceiling = models.BooleanField() + incumbent = models.BooleanField(default=False) + accepted_expenditure_ceiling = models.BooleanField(default=False) website = models.URLField(blank=True, null=True) twitter = models.URLField(blank=True, null=True) party_affiliation = models.CharField(blank=True, max_length=15) - occupation = models.CharField(blank=True, max_length=30) + occupation = models.CharField(blank=True, max_length=80) bio = models.TextField(blank=True) - photo = models.ImageField(blank=True, null=True) + # TODO use ImageField to support photo uploads + photo = models.URLField(blank=True, null=True) votersedge = models.URLField(blank=True, null=True) + def __str__(self): + return self.candidate + class Referendum(models.Model): measure_number = models.CharField(max_length=5) diff --git a/gsheets/parsers.py b/gsheets/parsers.py new file mode 100644 index 0000000..e69de29 diff --git a/gsheets/serializers.py b/gsheets/serializers.py new file mode 100644 index 0000000..8a85239 --- /dev/null +++ b/gsheets/serializers.py @@ -0,0 +1,10 @@ +from rest_framework import serializers +from .models import Candidate + + +class CandidateSerializer(serializers.ModelSerializer): + fppc = serializers.IntegerField(allow_null=True) + + class Meta: + model = Candidate + fields = '__all__' diff --git a/gsheets/views.py b/gsheets/views.py index 91ea44a..f63d8c5 100644 --- a/gsheets/views.py +++ b/gsheets/views.py @@ -1,3 +1,11 @@ -from django.shortcuts import render +from rest_framework import viewsets +from .models import Candidate +from .serializers import CandidateSerializer -# Create your views here. + +class CandidateViewSet(viewsets.ReadOnlyModelViewSet): + """ + API endpoint that allows Candidates to be viewed. + """ + queryset = Candidate.objects.all() + serializer_class = CandidateSerializer From 21cce03604182275f2541ea5965b2805445a4b1b Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 16:38:06 -0800 Subject: [PATCH 03/20] Configure logger --- ballot_api/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ballot_api/settings.py b/ballot_api/settings.py index 00653f3..c835334 100644 --- a/ballot_api/settings.py +++ b/ballot_api/settings.py @@ -1,5 +1,8 @@ +import logging import os +logging.basicConfig(level=logging.INFO) + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = 'y%72v=z^op&=n4$ye^u$*n0!a=v*-qs_&)#rs*wf+m3up85)(#' From 818da556b06333ae3fbfac0581972405620a87ab Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 18:57:03 -0800 Subject: [PATCH 04/20] Refactor into Parser --- gsheets/management/commands/gsheets_import.py | 67 +++++++++---------- gsheets/parsers.py | 66 ++++++++++++++++++ 2 files changed, 98 insertions(+), 35 deletions(-) diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py index 886eb26..3a1bbcc 100644 --- a/gsheets/management/commands/gsheets_import.py +++ b/gsheets/management/commands/gsheets_import.py @@ -8,6 +8,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand +from gsheets import parsers from gsheets.models import Candidate from gsheets.serializers import CandidateSerializer @@ -31,10 +32,25 @@ def add_arguments(self, parser): default=False, help="Import the entity even if it already exists.") + def format_csv_fields(self, row): + """Lowercases the keys of row and removes any special characters""" + + # Lowercase the keys + model = ((k.lower(), v) for k, v in row.items()) + + # Replace spaces with underscore + model = ((re.sub(r'\s+', '_', k), v) for k, v in model) + + # Strip other characters + model = ((re.sub(r'[^a-z_]', '', k), v) for k, v in model) + + return dict(model) + def handle(self, *args, **options): - force = options['force'] + self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options) - with urllib.request.urlopen(CANDIDATES_SHEET) as request: + def fetch_and_parse_from_url(self, url, parser, force=False, **options): + with urllib.request.urlopen(url) as request: with tempfile.TemporaryFile() as csvfile: csvfile.write(request.read()) csvfile.seek(0) @@ -45,50 +61,31 @@ def handle(self, *args, **options): import_errors = 0 for row in reader: total += 1 - name = row.get('Candidate', None) - if not name: - continue - exists = True - try: - Candidate.objects.get(candidate=name) - except ObjectDoesNotExist: - exists = False + row = self.format_csv_fields(row) + logger.debug(row) + exists = parser.exists_in_db(row) if exists and not force: + # Skip this row continue - field_names = [field.name for field in Candidate._meta.get_fields()] - model = ((k.lower(), v) for k, v in row.items()) - model = ((re.sub(r'\s+', '_', k), v) for k, v in model) - model = ((re.sub(r'[^a-z_]', '', k), v) for k, v in model) - model = dict((k, v) for k, v in model if k in field_names) - - # Convert fppc - fppc = model.get('fppc', None) - if not fppc: - model['fppc'] = None - - # Parse booleans - accepted_expenditure_ceiling = model.get('accepted_expenditure_ceiling', False) - model['accepted_expenditure_ceiling'] = bool(accepted_expenditure_ceiling) - - # Convert twitter @handle to URL - twitter = model.get('twitter', None) - if twitter: - # Drop the first char (@) - model['twitter'] = 'https://twitter.com/%s' % twitter[1:] + model = parser.parse(row) logger.debug(model) - serializer = CandidateSerializer(data=model) + + serializer = parser.to_serializer(model) if not serializer.is_valid(): import_errors += 1 - logger.error('Candidate could not be parsed: name="%s" parse_errors=%s row=%s', - name, serializer.errors, model) + logger.error('%s could not be parsed: parse_errors=%s row=%s', + parser.name(), serializer.errors, model) continue - Candidate.objects.update_or_create(candidate=name, defaults=serializer.validated_data) + obj, created = parser.commit(serializer) imported += 1 - logger.info('Imported candidate "%s"' % name) + if created: + logger.info('Created %s', parser.name()) + else: + logger.info('Updated %s', parser.name()) logger.info('Import candidates complete: total=%s imported=%s errors=%s', total, imported, import_errors) diff --git a/gsheets/parsers.py b/gsheets/parsers.py index e69de29..58c10fb 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -0,0 +1,66 @@ +from django.core.exceptions import ObjectDoesNotExist + +from .models import Candidate +from .serializers import CandidateSerializer + + +class Parser(object): + def name(self): + return self.__class__.model.__name__ + + def filter_fields(self, row): + field_names = self.get_field_names() + return dict((k, v) for k, v in row.items() if k in field_names) + + def parse(self, row): + return self.parse_fields(self.filter_fields(row)) + + +class CandidateParser(Parser): + model = Candidate + + def key(self, row): + return row.get('candidate', None) + + def exists_in_db(self, row): + """Determines if this row exists in the database already.""" + + name = row.get('candidate', None) + if not name: + return False + + exists = True + try: + Candidate.objects.get(candidate=name) + except ObjectDoesNotExist: + exists = False + + return exists + + def parse_fields(self, row): + """Parses individual fields in the row that are acceptable to the model.""" + # Convert fppc + fppc = row.get('fppc', None) + if not fppc: + row['fppc'] = None + + # Parse booleans + accepted_expenditure_ceiling = row.get('accepted_expenditure_ceiling', False) + row['accepted_expenditure_ceiling'] = bool(accepted_expenditure_ceiling) + + # Convert twitter @handle to URL + twitter = row.get('twitter', None) + if twitter: + # Drop the first char (@) + row['twitter'] = 'https://twitter.com/%s' % twitter[1:] + + return row + + def get_field_names(self): + return [field.name for field in Candidate._meta.get_fields()] + + def to_serializer(self, row): + return CandidateSerializer(data=row) + + def commit(self, serializer): + return Candidate.objects.update_or_create(candidate=serializer.data.get('candidate'), defaults=serializer.validated_data) From 3f51e788299018f1576a5ebcc242ce80e5807b1d Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 19:38:43 -0800 Subject: [PATCH 05/20] Add import for Committee --- gsheets/management/commands/gsheets_import.py | 17 +++-- gsheets/models.py | 9 ++- gsheets/parsers.py | 63 ++++++++++++++----- gsheets/serializers.py | 9 ++- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py index 3a1bbcc..6fc0dc3 100644 --- a/gsheets/management/commands/gsheets_import.py +++ b/gsheets/management/commands/gsheets_import.py @@ -5,12 +5,9 @@ import tempfile import urllib.request -from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand from gsheets import parsers -from gsheets.models import Candidate -from gsheets.serializers import CandidateSerializer CANDIDATES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=csv' # noqa @@ -48,6 +45,7 @@ def format_csv_fields(self, row): def handle(self, *args, **options): self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options) + self.fetch_and_parse_from_url(COMMITTEES_SHEET, parsers.CommitteeParser(), **options) def fetch_and_parse_from_url(self, url, parser, force=False, **options): with urllib.request.urlopen(url) as request: @@ -70,22 +68,23 @@ def fetch_and_parse_from_url(self, url, parser, force=False, **options): # Skip this row continue + id = row.get(parser.key) model = parser.parse(row) logger.debug(model) serializer = parser.to_serializer(model) if not serializer.is_valid(): import_errors += 1 - logger.error('%s could not be parsed: parse_errors=%s row=%s', - parser.name(), serializer.errors, model) + logger.error('%s "%s" could not be parsed: parse_errors=%s row=%s', + parser.name, id, serializer.errors, model) continue obj, created = parser.commit(serializer) imported += 1 if created: - logger.info('Created %s', parser.name()) + logger.info('Created %s "%s"', parser.name, id) else: - logger.info('Updated %s', parser.name()) + logger.info('Updated %s "%s"', parser.name, id) - logger.info('Import candidates complete: total=%s imported=%s errors=%s', - total, imported, import_errors) + logger.info('Import %s data complete: total=%s imported=%s errors=%s', + parser.name, total, imported, import_errors) diff --git a/gsheets/models.py b/gsheets/models.py index 1f52b74..7d3f795 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -40,10 +40,10 @@ class Committee(models.Model): ('O', 'Oppose'), ] - filer_id = models.IntegerField() - filer_naml = models.CharField(blank=True, max_length=20) + filer_id = models.CharField(max_length=15) + filer_naml = models.CharField(blank=True, max_length=200) committee_type = models.CharField(blank=True, max_length=3, choices=COMMITTEE_TYPES) - description = models.CharField(blank=True, max_length=20) + description = models.CharField(blank=True, max_length=40) ballot_measure = models.CharField(blank=True, max_length=3) support_or_oppose = models.CharField(blank=True, max_length=1, choices=SUPPORT_OPPOSE) website = models.URLField(blank=True, null=True) @@ -51,6 +51,9 @@ class Committee(models.Model): facebook = models.URLField(blank=True, null=True) netfilelocalid = models.CharField(blank=True, max_length=15) + def __str__(self): + return self.filer_naml + class ReferendumMapping(models.Model): measure_name = models.CharField(max_length=30) diff --git a/gsheets/parsers.py b/gsheets/parsers.py index 58c10fb..a953408 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -1,12 +1,13 @@ from django.core.exceptions import ObjectDoesNotExist -from .models import Candidate -from .serializers import CandidateSerializer +from .models import Candidate, Committee +from .serializers import CandidateSerializer, CommitteeSerializer class Parser(object): + @property def name(self): - return self.__class__.model.__name__ + return self.model.__name__ def filter_fields(self, row): field_names = self.get_field_names() @@ -15,28 +16,48 @@ def filter_fields(self, row): def parse(self, row): return self.parse_fields(self.filter_fields(row)) + def parse_fields(self, row): + """Subclasses should override this method to transform the row to appropriate types""" -class CandidateParser(Parser): - model = Candidate + return row - def key(self, row): - return row.get('candidate', None) + def get_field_names(self): + return [field.name for field in self.model._meta.get_fields()] def exists_in_db(self, row): """Determines if this row exists in the database already.""" - name = row.get('candidate', None) - if not name: + id = row.get(self.key, None) + if not id: return False + kwargs = {self.key: id} exists = True try: - Candidate.objects.get(candidate=name) + self.model.objects.get(**kwargs) except ObjectDoesNotExist: exists = False return exists + def to_serializer(self, row): + return self.__class__.serializer(data=row) + + def commit(self, serializer): + id = serializer.data.get(self.key) + kwargs = { + self.key: id, + "defaults": serializer.validated_data + } + + return self.model.objects.update_or_create(**kwargs) + + +class CandidateParser(Parser): + model = Candidate + key = 'candidate' + serializer = CandidateSerializer + def parse_fields(self, row): """Parses individual fields in the row that are acceptable to the model.""" # Convert fppc @@ -56,11 +77,21 @@ def parse_fields(self, row): return row - def get_field_names(self): - return [field.name for field in Candidate._meta.get_fields()] - def to_serializer(self, row): - return CandidateSerializer(data=row) +class CommitteeParser(Parser): + model = Committee + key = 'filer_id' + serializer = CommitteeSerializer - def commit(self, serializer): - return Candidate.objects.update_or_create(candidate=serializer.data.get('candidate'), defaults=serializer.validated_data) + def parse_fields(self, row): + twitter = row.get('twitter', None) + if twitter: + # Drop the first char (@) + twitter = 'https://twitter.com/%s' % twitter[1:] + + parsed = dict(row) + parsed.update( + twitter=twitter, + ) + + return parsed diff --git a/gsheets/serializers.py b/gsheets/serializers.py index 8a85239..a719996 100644 --- a/gsheets/serializers.py +++ b/gsheets/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from .models import Candidate +from .models import Candidate, Committee class CandidateSerializer(serializers.ModelSerializer): @@ -8,3 +8,10 @@ class CandidateSerializer(serializers.ModelSerializer): class Meta: model = Candidate fields = '__all__' + + +class CommitteeSerializer(serializers.ModelSerializer): + + class Meta: + model = Committee + fields = '__all__' From fa1575db71317c00f952bf653d026eb04558f986 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 19:43:44 -0800 Subject: [PATCH 06/20] Import Referendums --- gsheets/management/commands/gsheets_import.py | 1 + gsheets/models.py | 5 ++++- gsheets/parsers.py | 10 ++++++++-- gsheets/serializers.py | 9 ++++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py index 6fc0dc3..144da04 100644 --- a/gsheets/management/commands/gsheets_import.py +++ b/gsheets/management/commands/gsheets_import.py @@ -46,6 +46,7 @@ def format_csv_fields(self, row): def handle(self, *args, **options): self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options) self.fetch_and_parse_from_url(COMMITTEES_SHEET, parsers.CommitteeParser(), **options) + self.fetch_and_parse_from_url(REFERENDUMS_SHEET, parsers.ReferendumParser(), **options) def fetch_and_parse_from_url(self, url, parser, force=False, **options): with urllib.request.urlopen(url) as request: diff --git a/gsheets/models.py b/gsheets/models.py index 7d3f795..27a3985 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -23,11 +23,14 @@ def __str__(self): class Referendum(models.Model): measure_number = models.CharField(max_length=5) - short_title = models.CharField(blank=True, max_length=30) + short_title = models.CharField(blank=True, max_length=200) full_title = models.TextField(blank=True) summary = models.TextField() votersedge = models.URLField(blank=True, null=True) + def __str__(self): + return "Referendum %s %s" % (self.measure_number, self.short_title) + class Committee(models.Model): COMMITTEE_TYPES = [ diff --git a/gsheets/parsers.py b/gsheets/parsers.py index a953408..d75f089 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -1,7 +1,7 @@ from django.core.exceptions import ObjectDoesNotExist -from .models import Candidate, Committee -from .serializers import CandidateSerializer, CommitteeSerializer +from .models import Candidate, Committee, Referendum +from .serializers import CandidateSerializer, CommitteeSerializer, ReferendumSerializer class Parser(object): @@ -95,3 +95,9 @@ def parse_fields(self, row): ) return parsed + + +class ReferendumParser(Parser): + model = Referendum + key = 'measure_number' + serializer = ReferendumSerializer diff --git a/gsheets/serializers.py b/gsheets/serializers.py index a719996..d62958a 100644 --- a/gsheets/serializers.py +++ b/gsheets/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from .models import Candidate, Committee +from .models import Candidate, Committee, Referendum class CandidateSerializer(serializers.ModelSerializer): @@ -15,3 +15,10 @@ class CommitteeSerializer(serializers.ModelSerializer): class Meta: model = Committee fields = '__all__' + + +class ReferendumSerializer(serializers.ModelSerializer): + + class Meta: + model = Referendum + fields = '__all__' From 515d35fbc0df83841eee796ee9f5ec244b536c8a Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sat, 4 Feb 2017 20:40:17 -0800 Subject: [PATCH 07/20] Import referendum mappings --- gsheets/management/commands/gsheets_import.py | 1 + gsheets/models.py | 2 +- gsheets/parsers.py | 10 ++++++++-- gsheets/serializers.py | 9 ++++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py index 144da04..605fabf 100644 --- a/gsheets/management/commands/gsheets_import.py +++ b/gsheets/management/commands/gsheets_import.py @@ -47,6 +47,7 @@ def handle(self, *args, **options): self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options) self.fetch_and_parse_from_url(COMMITTEES_SHEET, parsers.CommitteeParser(), **options) self.fetch_and_parse_from_url(REFERENDUMS_SHEET, parsers.ReferendumParser(), **options) + self.fetch_and_parse_from_url(REFERENDUM_NAME_TO_NUMBER_SHEET, parsers.ReferendumMappingParser(), **options) def fetch_and_parse_from_url(self, url, parser, force=False, **options): with urllib.request.urlopen(url) as request: diff --git a/gsheets/models.py b/gsheets/models.py index 27a3985..ca3dcae 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -59,5 +59,5 @@ def __str__(self): class ReferendumMapping(models.Model): - measure_name = models.CharField(max_length=30) + measure_name = models.CharField(max_length=200) measure_number = models.CharField(max_length=3) diff --git a/gsheets/parsers.py b/gsheets/parsers.py index d75f089..ce87f75 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -1,7 +1,7 @@ from django.core.exceptions import ObjectDoesNotExist -from .models import Candidate, Committee, Referendum -from .serializers import CandidateSerializer, CommitteeSerializer, ReferendumSerializer +from .models import Candidate, Committee, Referendum, ReferendumMapping +from .serializers import CandidateSerializer, CommitteeSerializer, ReferendumSerializer, ReferendumMappingSerializer class Parser(object): @@ -101,3 +101,9 @@ class ReferendumParser(Parser): model = Referendum key = 'measure_number' serializer = ReferendumSerializer + + +class ReferendumMappingParser(Parser): + model = ReferendumMapping + key = 'measure_number' + serializer = ReferendumMappingSerializer diff --git a/gsheets/serializers.py b/gsheets/serializers.py index d62958a..11562ce 100644 --- a/gsheets/serializers.py +++ b/gsheets/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from .models import Candidate, Committee, Referendum +from .models import Candidate, Committee, Referendum, ReferendumMapping class CandidateSerializer(serializers.ModelSerializer): @@ -22,3 +22,10 @@ class ReferendumSerializer(serializers.ModelSerializer): class Meta: model = Referendum fields = '__all__' + + +class ReferendumMappingSerializer(serializers.ModelSerializer): + + class Meta: + model = ReferendumMapping + fields = '__all__' From ee78244eb0439356640b2bde17544d0124f98adf Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 5 Feb 2017 00:06:15 -0800 Subject: [PATCH 08/20] Import candidate aliases --- gsheets/management/commands/gsheets_import.py | 13 ++-- gsheets/models.py | 13 ++++ gsheets/parsers.py | 66 +++++++++++++++---- 3 files changed, 73 insertions(+), 19 deletions(-) diff --git a/gsheets/management/commands/gsheets_import.py b/gsheets/management/commands/gsheets_import.py index 605fabf..1a6b433 100644 --- a/gsheets/management/commands/gsheets_import.py +++ b/gsheets/management/commands/gsheets_import.py @@ -71,17 +71,18 @@ def fetch_and_parse_from_url(self, url, parser, force=False, **options): continue id = row.get(parser.key) - model = parser.parse(row) - logger.debug(model) - serializer = parser.to_serializer(model) - if not serializer.is_valid(): + try: + data = parser.parse(row) + logger.debug(data) + model, created = parser.commit(data) + except Exception as err: import_errors += 1 logger.error('%s "%s" could not be parsed: parse_errors=%s row=%s', - parser.name, id, serializer.errors, model) + parser.name, id, err, row) + logger.exception(err) continue - obj, created = parser.commit(serializer) imported += 1 if created: logger.info('Created %s "%s"', parser.name, id) diff --git a/gsheets/models.py b/gsheets/models.py index ca3dcae..9874214 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -1,6 +1,19 @@ from django.db import models +class CandidateAlias(models.Model): + """An alternative name for the candidate which may occur in filings.""" + + candidate_alias = models.CharField(max_length=30) + candidate = models.ForeignKey('Candidate', on_delete=models.CASCADE, related_name='aliases') + + def __str__(self): + return self.candidate_alias + + class Meta: + verbose_name_plural = 'candidate aliases' + + class Candidate(models.Model): fppc = models.IntegerField(blank=True, null=True, unique=True) committee_name = models.CharField(blank=True, max_length=120) diff --git a/gsheets/parsers.py b/gsheets/parsers.py index ce87f75..565714e 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -1,9 +1,18 @@ from django.core.exceptions import ObjectDoesNotExist -from .models import Candidate, Committee, Referendum, ReferendumMapping +from .models import Candidate, CandidateAlias, Committee, Referendum, ReferendumMapping from .serializers import CandidateSerializer, CommitteeSerializer, ReferendumSerializer, ReferendumMappingSerializer +def get_or_none(key, row): + value = row.get(key, None) + # Convert falsy values to None + if not value: + value = None + + return value + + class Parser(object): @property def name(self): @@ -43,11 +52,11 @@ def exists_in_db(self, row): def to_serializer(self, row): return self.__class__.serializer(data=row) - def commit(self, serializer): - id = serializer.data.get(self.key) + def commit(self, data): + id = data.get(self.key) kwargs = { self.key: id, - "defaults": serializer.validated_data + "defaults": data } return self.model.objects.update_or_create(**kwargs) @@ -58,23 +67,54 @@ class CandidateParser(Parser): key = 'candidate' serializer = CandidateSerializer - def parse_fields(self, row): + def get_field_names(self): + return super(CandidateParser, self).get_field_names() + ['aliases'] + + def parse(self, row): """Parses individual fields in the row that are acceptable to the model.""" - # Convert fppc - fppc = row.get('fppc', None) - if not fppc: - row['fppc'] = None - # Parse booleans - accepted_expenditure_ceiling = row.get('accepted_expenditure_ceiling', False) - row['accepted_expenditure_ceiling'] = bool(accepted_expenditure_ceiling) + data = self.filter_fields(row) + + # We'll handle aliases below, don't pass them to the eventual model + del data['aliases'] + + # Convert empty strings to None + fppc = get_or_none('fppc', row) + + # Convert empty strings to bool + accepted_expenditure_ceiling = bool(row.get('accepted_expenditure_ceiling', False)) # Convert twitter @handle to URL twitter = row.get('twitter', None) if twitter: # Drop the first char (@) - row['twitter'] = 'https://twitter.com/%s' % twitter[1:] + twitter = 'https://twitter.com/%s' % twitter[1:] + + data.update( + fppc=fppc, + accepted_expenditure_ceiling=accepted_expenditure_ceiling, + twitter=twitter) + + candidate, created = self.commit(data) + + # Parse aliases + alias_parser = CandidateAliasParser() + candidate_aliases = row.get('aliases', '').split(',') + for candidate_alias in candidate_aliases: + if not candidate_alias: + continue + + alias = alias_parser.parse(dict(candidate_alias=candidate_alias, candidate=candidate)) + alias_parser.commit(alias) + return data + + +class CandidateAliasParser(Parser): + model = CandidateAlias + key = 'candidate_alias' + + def parse(self, row): return row From 95112cb92dd5b6275ab9a3bea8d79d26bd21cb52 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 5 Feb 2017 00:07:31 -0800 Subject: [PATCH 09/20] Include candidate aliases in admin --- gsheets/admin.py | 14 ++++++++++++-- gsheets/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/gsheets/admin.py b/gsheets/admin.py index fe6b360..a329755 100644 --- a/gsheets/admin.py +++ b/gsheets/admin.py @@ -1,8 +1,18 @@ from django.contrib import admin -from .models import Candidate, Committee, Referendum, ReferendumMapping +from .models import Candidate, CandidateAlias, Committee, Referendum, ReferendumMapping -admin.site.register(Candidate) + +class CandidateAliasInline(admin.TabularInline): + model = CandidateAlias + + +class CandidateAdmin(admin.ModelAdmin): + inlines = (CandidateAliasInline,) + + +admin.site.register(Candidate, CandidateAdmin) +admin.site.register(CandidateAlias) admin.site.register(Referendum) admin.site.register(ReferendumMapping) admin.site.register(Committee) diff --git a/gsheets/models.py b/gsheets/models.py index 9874214..12ac023 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -17,7 +17,7 @@ class Meta: class Candidate(models.Model): fppc = models.IntegerField(blank=True, null=True, unique=True) committee_name = models.CharField(blank=True, max_length=120) - candidate = models.CharField(max_length=30) + candidate = models.CharField(max_length=30, help_text='The candidate\'s full name.') office = models.CharField(blank=True, max_length=30, help_text='Office the candidate is running for.') incumbent = models.BooleanField(default=False) accepted_expenditure_ceiling = models.BooleanField(default=False) From cf3bc0840b1216bcf0b88ab7e75b304098af2a31 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 21 Mar 2017 19:19:47 -0700 Subject: [PATCH 10/20] Run app on heroku - Configuration for postgres - Use whitenoise for static files - Specify python 3.6.0 - Run gunicorn for wsgi --- Procfile | 1 + ballot_api/settings/__init__.py | 1 + .../{settings.py => settings/common.py} | 24 +++++++++++++------ ballot_api/settings/heroku.py | 5 ++++ requirements.txt | 4 ++++ runtime.txt | 1 + 6 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 Procfile create mode 100644 ballot_api/settings/__init__.py rename ballot_api/{settings.py => settings/common.py} (81%) create mode 100644 ballot_api/settings/heroku.py create mode 100644 runtime.txt diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..511c858 --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn ballot_api.wsgi --log-file - diff --git a/ballot_api/settings/__init__.py b/ballot_api/settings/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ballot_api/settings/__init__.py @@ -0,0 +1 @@ + diff --git a/ballot_api/settings.py b/ballot_api/settings/common.py similarity index 81% rename from ballot_api/settings.py rename to ballot_api/settings/common.py index 779755d..be2ca46 100644 --- a/ballot_api/settings.py +++ b/ballot_api/settings/common.py @@ -8,6 +8,15 @@ ALLOWED_HOSTS = [] +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'ballot_api', + 'HOST': '127.0.0.1', + 'PORT': '5432', + } +} + INSTALLED_APPS = ( 'flat', 'django.contrib.admin', @@ -31,6 +40,7 @@ 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', ) ROOT_URLCONF = 'ballot_api.urls' @@ -53,13 +63,6 @@ WSGI_APPLICATION = 'ballot_api.wsgi.application' -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' @@ -71,3 +74,10 @@ USE_TZ = True STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' + +try: + from local_settings import * +except ImportError: + pass diff --git a/ballot_api/settings/heroku.py b/ballot_api/settings/heroku.py new file mode 100644 index 0000000..0503a1d --- /dev/null +++ b/ballot_api/settings/heroku.py @@ -0,0 +1,5 @@ +from .common import * +import dj_database_url + + +DATABASES['default'] = dj_database_url.config() diff --git a/requirements.txt b/requirements.txt index f2e1e55..5a150d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,22 @@ coreapi==2.1.1 +dj-database-url==0.4.2 Django==1.8 django-flat-theme==1.1.4 django-rest-swagger==2.1.1 djangorestframework==3.5.3 docutils==0.13.1 flake8==3.2.1 +gunicorn==19.7.1 itypes==1.1.0 mccabe==0.5.3 olefile==0.44 openapi-codec==1.2.1 pep8==1.7.0 Pillow==4.0.0 +psycopg2==2.7.1 pycodestyle==2.2.0 pyflakes==1.3.0 requests==2.13.0 simplejson==3.10.0 uritemplate==3.0.0 +whitenoise==3.3.0 diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..3a799ff --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.6.0 From e1dc6c7f2e4b7a3bc5c8b58dde1a6f23d3e027a1 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 28 Mar 2017 19:46:40 -0700 Subject: [PATCH 11/20] Rearrange settings to be easier for development --- ballot_api/settings/common.py | 5 ----- ballot_api/settings/heroku.py | 10 +++++++++- ballot_api/wsgi.py | 2 +- manage.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ballot_api/settings/common.py b/ballot_api/settings/common.py index be2ca46..ac3e2f5 100644 --- a/ballot_api/settings/common.py +++ b/ballot_api/settings/common.py @@ -76,8 +76,3 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' - -try: - from local_settings import * -except ImportError: - pass diff --git a/ballot_api/settings/heroku.py b/ballot_api/settings/heroku.py index 0503a1d..f577892 100644 --- a/ballot_api/settings/heroku.py +++ b/ballot_api/settings/heroku.py @@ -1,5 +1,13 @@ -from .common import * +import os + import dj_database_url +from .common import * + DATABASES['default'] = dj_database_url.config() + +SECRET_KEY = os.environ.get('SECRET_KEY', None) + +if not SECRET_KEY: + raise Exception('SECRET_KEY environment variable must be set for heroku configuration.') diff --git a/ballot_api/wsgi.py b/ballot_api/wsgi.py index 2b89bb0..644933c 100644 --- a/ballot_api/wsgi.py +++ b/ballot_api/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings.heroku") application = get_wsgi_application() diff --git a/manage.py b/manage.py index eca1dce..c45d528 100644 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ballot_api.settings.development") try: from django.core.management import execute_from_command_line except ImportError: From 1229c1f0310f1b58db9070610f37fca82330bd8b Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 28 Mar 2017 19:47:02 -0700 Subject: [PATCH 12/20] Note the PostgreSQL dependency --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fefe7eb..2f04454 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ California. ## Prequisites +- [PostgreSQL](https://www.postgresql.org/) + ## Setup From 42dd17c08ad64c16aaba3237e17ef4e68f4e9242 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 28 Mar 2017 20:03:29 -0700 Subject: [PATCH 13/20] Linting exceptions for settings --- .travis.yml | 2 ++ ballot_api/settings/__init__.py | 1 - ballot_api/settings/heroku.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index abbeb93..02cb7de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ language: python +python: + - 3.6 script: flake8 diff --git a/ballot_api/settings/__init__.py b/ballot_api/settings/__init__.py index 8b13789..e69de29 100644 --- a/ballot_api/settings/__init__.py +++ b/ballot_api/settings/__init__.py @@ -1 +0,0 @@ - diff --git a/ballot_api/settings/heroku.py b/ballot_api/settings/heroku.py index f577892..71b53a6 100644 --- a/ballot_api/settings/heroku.py +++ b/ballot_api/settings/heroku.py @@ -1,3 +1,5 @@ +# flake8: noqa + import os import dj_database_url From 4d430698641f51e99010faf615b1c76b51c0df40 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 28 Mar 2017 21:08:25 -0700 Subject: [PATCH 14/20] Parse party affiliation --- gsheets/models.py | 16 ++++++++++++++-- gsheets/parsers.py | 17 +++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/gsheets/models.py b/gsheets/models.py index 12ac023..728cb95 100644 --- a/gsheets/models.py +++ b/gsheets/models.py @@ -15,6 +15,15 @@ class Meta: class Candidate(models.Model): + """A person running for office in an election.""" + + PARTY_AFFILIATION = [ + ('D', 'Democrat'), + ('R', 'Republican'), + ('I', 'Independent'), + ('O', 'Other'), + ] + fppc = models.IntegerField(blank=True, null=True, unique=True) committee_name = models.CharField(blank=True, max_length=120) candidate = models.CharField(max_length=30, help_text='The candidate\'s full name.') @@ -23,7 +32,7 @@ class Candidate(models.Model): accepted_expenditure_ceiling = models.BooleanField(default=False) website = models.URLField(blank=True, null=True) twitter = models.URLField(blank=True, null=True) - party_affiliation = models.CharField(blank=True, max_length=15) + party_affiliation = models.CharField(blank=True, max_length=1, choices=PARTY_AFFILIATION) occupation = models.CharField(blank=True, max_length=80) bio = models.TextField(blank=True) # TODO use ImageField to support photo uploads @@ -73,4 +82,7 @@ def __str__(self): class ReferendumMapping(models.Model): measure_name = models.CharField(max_length=200) - measure_number = models.CharField(max_length=3) + measure_number = models.CharField(max_length=10) + + def __str__(self): + return '%s: %s' % (self.measure_number, self.measure_name) diff --git a/gsheets/parsers.py b/gsheets/parsers.py index 565714e..59bf3c9 100644 --- a/gsheets/parsers.py +++ b/gsheets/parsers.py @@ -67,8 +67,17 @@ class CandidateParser(Parser): key = 'candidate' serializer = CandidateSerializer - def get_field_names(self): - return super(CandidateParser, self).get_field_names() + ['aliases'] + def parse_party_affiliation(self, row): + party_affiliation = row.get('party_affiliation', None) + if not party_affiliation: + return 'O' + + # Take only the first character + party_affiliation = party_affiliation[0] + if party_affiliation not in [k for k, _ in Candidate.PARTY_AFFILIATION]: + return 'O' + + return party_affiliation def parse(self, row): """Parses individual fields in the row that are acceptable to the model.""" @@ -84,6 +93,9 @@ def parse(self, row): # Convert empty strings to bool accepted_expenditure_ceiling = bool(row.get('accepted_expenditure_ceiling', False)) + # Party affiliation + party_affiliation = self.parse_party_affiliation(row) + # Convert twitter @handle to URL twitter = row.get('twitter', None) if twitter: @@ -93,6 +105,7 @@ def parse(self, row): data.update( fppc=fppc, accepted_expenditure_ceiling=accepted_expenditure_ceiling, + party_affiliation=party_affiliation, twitter=twitter) candidate, created = self.commit(data) From 61062b12b4bf498431fe8c7cc2e98fc8a95ca872 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Tue, 28 Mar 2017 21:09:04 -0700 Subject: [PATCH 15/20] Include gsheets migrations --- gsheets/migrations/0001_initial.py | 67 +++++++++++++++++++ gsheets/migrations/0002_auto_20170205_0455.py | 32 +++++++++ gsheets/migrations/0003_auto_20170205_0802.py | 28 ++++++++ gsheets/migrations/0004_auto_20170329_0408.py | 19 ++++++ 4 files changed, 146 insertions(+) create mode 100644 gsheets/migrations/0001_initial.py create mode 100644 gsheets/migrations/0002_auto_20170205_0455.py create mode 100644 gsheets/migrations/0003_auto_20170205_0802.py create mode 100644 gsheets/migrations/0004_auto_20170329_0408.py diff --git a/gsheets/migrations/0001_initial.py b/gsheets/migrations/0001_initial.py new file mode 100644 index 0000000..ce48ce8 --- /dev/null +++ b/gsheets/migrations/0001_initial.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Candidate', + fields=[ + ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)), + ('fppc', models.IntegerField(blank=True, unique=True, null=True)), + ('committee_name', models.CharField(blank=True, max_length=120)), + ('candidate', models.CharField(max_length=30)), + ('office', models.CharField(blank=True, help_text='Office the candidate is running for.', max_length=30)), + ('incumbent', models.BooleanField(default=False)), + ('accepted_expenditure_ceiling', models.BooleanField(default=False)), + ('website', models.URLField(blank=True, null=True)), + ('twitter', models.URLField(blank=True, null=True)), + ('party_affiliation', models.CharField(blank=True, max_length=15)), + ('occupation', models.CharField(blank=True, max_length=80)), + ('bio', models.TextField(blank=True)), + ('photo', models.URLField(blank=True, null=True)), + ('votersedge', models.URLField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='Committee', + fields=[ + ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)), + ('filer_id', models.CharField(max_length=15)), + ('filer_naml', models.CharField(blank=True, max_length=200)), + ('committee_type', models.CharField(choices=[('RCP', 'RCP'), ('BMC', 'BMC')], blank=True, max_length=3)), + ('description', models.CharField(blank=True, max_length=40)), + ('ballot_measure', models.CharField(blank=True, max_length=3)), + ('support_or_oppose', models.CharField(choices=[('S', 'Support'), ('O', 'Oppose')], blank=True, max_length=1)), + ('website', models.URLField(blank=True, null=True)), + ('twitter', models.URLField(blank=True, null=True)), + ('facebook', models.URLField(blank=True, null=True)), + ('netfilelocalid', models.CharField(blank=True, max_length=15)), + ], + ), + migrations.CreateModel( + name='Referendum', + fields=[ + ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)), + ('measure_number', models.CharField(max_length=5)), + ('short_title', models.CharField(blank=True, max_length=30)), + ('full_title', models.TextField(blank=True)), + ('summary', models.TextField()), + ('votersedge', models.URLField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='ReferendumMapping', + fields=[ + ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)), + ('measure_name', models.CharField(max_length=30)), + ('measure_number', models.CharField(max_length=3)), + ], + ), + ] diff --git a/gsheets/migrations/0002_auto_20170205_0455.py b/gsheets/migrations/0002_auto_20170205_0455.py new file mode 100644 index 0000000..aace0a2 --- /dev/null +++ b/gsheets/migrations/0002_auto_20170205_0455.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gsheets', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='CandidateAlias', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), + ('candidate_alias', models.CharField(max_length=30)), + ('candidate', models.ForeignKey(to='gsheets.Candidate', related_name='aliases')), + ], + ), + migrations.AlterField( + model_name='referendum', + name='short_title', + field=models.CharField(max_length=200, blank=True), + ), + migrations.AlterField( + model_name='referendummapping', + name='measure_name', + field=models.CharField(max_length=200), + ), + ] diff --git a/gsheets/migrations/0003_auto_20170205_0802.py b/gsheets/migrations/0003_auto_20170205_0802.py new file mode 100644 index 0000000..423cc3b --- /dev/null +++ b/gsheets/migrations/0003_auto_20170205_0802.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gsheets', '0002_auto_20170205_0455'), + ] + + operations = [ + migrations.AlterModelOptions( + name='candidatealias', + options={'verbose_name_plural': 'candidate aliases'}, + ), + migrations.AlterField( + model_name='candidate', + name='candidate', + field=models.CharField(max_length=30, help_text="The candidate's full name."), + ), + migrations.AlterField( + model_name='candidate', + name='party_affiliation', + field=models.CharField(max_length=1, choices=[('D', 'Democrat'), ('R', 'Republican'), ('I', 'Independent'), ('O', 'Other')], blank=True), + ), + ] diff --git a/gsheets/migrations/0004_auto_20170329_0408.py b/gsheets/migrations/0004_auto_20170329_0408.py new file mode 100644 index 0000000..a1d8366 --- /dev/null +++ b/gsheets/migrations/0004_auto_20170329_0408.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gsheets', '0003_auto_20170205_0802'), + ] + + operations = [ + migrations.AlterField( + model_name='referendummapping', + name='measure_number', + field=models.CharField(max_length=10), + ), + ] From 6bfd13f7532212b813b08adec961ffa1180ad14d Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 30 Mar 2017 08:02:30 -0700 Subject: [PATCH 16/20] Fix linting issues --- ballot/tests.py | 3 --- ballot_api/urls.py | 1 - gsheets/tests.py | 3 --- 3 files changed, 7 deletions(-) delete mode 100644 ballot/tests.py delete mode 100644 gsheets/tests.py diff --git a/ballot/tests.py b/ballot/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/ballot/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/ballot_api/urls.py b/ballot_api/urls.py index 7f64a4b..0819057 100644 --- a/ballot_api/urls.py +++ b/ballot_api/urls.py @@ -7,7 +7,6 @@ import gsheets.views as gsheets_views router = routers.SimpleRouter() -#router.register(r'candidates', ballot_views.CandidateViewSet) router.register(r'elections', ballot_views.ElectionViewSet) router.register(r'candidates', gsheets_views.CandidateViewSet) diff --git a/gsheets/tests.py b/gsheets/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/gsheets/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. From 374855155675a91404f2a247a411421d6073bfb5 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 9 Apr 2017 14:19:17 -0700 Subject: [PATCH 17/20] Include missing development settings --- ballot_api/settings/development.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ballot_api/settings/development.py diff --git a/ballot_api/settings/development.py b/ballot_api/settings/development.py new file mode 100644 index 0000000..0f5b26b --- /dev/null +++ b/ballot_api/settings/development.py @@ -0,0 +1,7 @@ +# flake8: noqa +from .common import * + +try: + from local_settings import * +except ImportError: + pass From 757f9069fbef151e7278db15499f0b35b5a0c7d2 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 9 Apr 2017 14:19:42 -0700 Subject: [PATCH 18/20] Fix staticfiles path Should be at the top level. Add it to gitignore. --- .gitignore | 1 + ballot_api/settings/common.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1d59fb6..5c05d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/ db.sqlite3 +/staticfiles/ diff --git a/ballot_api/settings/common.py b/ballot_api/settings/common.py index e7607fd..1c24b6e 100644 --- a/ballot_api/settings/common.py +++ b/ballot_api/settings/common.py @@ -1,6 +1,6 @@ import os -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) SECRET_KEY = 'y%72v=z^op&=n4$ye^u$*n0!a=v*-qs_&)#rs*wf+m3up85)(#' From 058542a4ceb73d3d601e8cd05a299883f86c6cda Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 9 Apr 2017 14:30:39 -0700 Subject: [PATCH 19/20] Include missing production settings --- ballot_api/settings/heroku.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ballot_api/settings/heroku.py b/ballot_api/settings/heroku.py index 71b53a6..6c4a277 100644 --- a/ballot_api/settings/heroku.py +++ b/ballot_api/settings/heroku.py @@ -6,6 +6,11 @@ from .common import * +DEBUG = False + +ALLOWED_HOSTS = [ + 'caciviclab-ballot-api.herokuapp.com', +] DATABASES['default'] = dj_database_url.config() From 099607baaecae530143dea77ad6874434bab92df Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Sun, 9 Apr 2017 15:52:59 -0700 Subject: [PATCH 20/20] Note python3 requirement --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f04454..1c8a0e5 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ California. ## Prequisites +- Python 3 - [PostgreSQL](https://www.postgresql.org/)