diff --git a/knesset_data_django/__init__.py b/knesset_data_django/__init__.py index 2b5bad0..a7bae1c 100644 --- a/knesset_data_django/__init__.py +++ b/knesset_data_django/__init__.py @@ -1,4 +1,5 @@ KNESSET_DATA_DJANGO_APPS = ( 'knesset_data_django.committees', 'knesset_data_django.common', + 'knesset_data_django.kns_persons', ) diff --git a/knesset_data_django/committees/models.py b/knesset_data_django/committees/models.py index b808c8d..fca26f5 100644 --- a/knesset_data_django/committees/models.py +++ b/knesset_data_django/committees/models.py @@ -1,2 +1,2 @@ # currently uses Open Knesset models. TODO: move to a separate data access layer -from committees.models import Committee, CommitteeMeeting, ProtocolPart +from committees.models import Committee, CommitteeMeeting, ProtocolPart, CommitteeMeetingAttendee diff --git a/knesset_data_django/committees/scrapers/committee_meeting_protocols.py b/knesset_data_django/committees/scrapers/committee_meeting_protocols.py index cfb99fe..fd667ce 100644 --- a/knesset_data_django/committees/scrapers/committee_meeting_protocols.py +++ b/knesset_data_django/committees/scrapers/committee_meeting_protocols.py @@ -1,7 +1,7 @@ from knesset_data_django.committees.protocol_part_builder import CommitteeProtocolPartBuilder from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper from ...mks.utils import get_all_mk_names -from ..models import CommitteeMeeting, ProtocolPart +from ..models import CommitteeMeeting, ProtocolPart, CommitteeMeetingAttendee from datetime import datetime import os import csv @@ -17,7 +17,7 @@ def _get_all_mk_names(self): setattr(self, 'loaded_mk_names', True) return self.mks, self.mk_names - def _validate_datapackage_item(self, text_file_path, parts_file_path, meetings_cnt): + def _validate_datapackage_item(self, text_file_path, parts_file_path, attendees_file_path, meetings_cnt): if meetings_cnt == 0: return False, "could not find meeting in DB" elif meetings_cnt > 1: @@ -26,6 +26,8 @@ def _validate_datapackage_item(self, text_file_path, parts_file_path, meetings_c return False, "missing text file due to error in datapackage creation, see scraper_errors field in datapackage" elif parts_file_path == self._get_datapackage_resource_path(): return False, "missing parts file due to error in datapackage creation, see scraper_errors field in datapackage" + elif attendees_file_path == self._get_datapackage_resource_path(): + return False, "missing attendees file due to error in datapackage creation, see scraper_errors field in datapackage" elif not os.path.exists(text_file_path): return False, "missing text file {}".format(text_file_path) elif not os.path.exists(parts_file_path): @@ -95,26 +97,50 @@ def _update_protocol_parts(self, meeting, parts_file_path): attended_ok, attended_message = False, "failed to updated attending members: {}".format(e) return protocol_ok, protocol_message, attended_ok, attended_message + def _save_attendees(self, meeting, attendees): + CommitteeMeetingAttendee.objects.bulk_create(attendees) + meeting.save() + + def _update_attendees(self, meeting, attendees_file_path): + attendees_ok, attendees_message = None, None + if meeting.attendees.count() > 0: + attendees_ok, attendees_message = False, "meeting has existing attendees, will not reparse" + else: + attendees_ok = True + with open(attendees_file_path) as f: + attendees = csv.reader(f) + assert attendees.next() == ['name','role','additional_information'] + committee_attendees = [CommitteeMeetingAttendee(comittee_meeting=meeting,name=attendee[0],role=attendee[1],additional_information=attendee[2] if len(attendee) > 2 else '') + for attendee in enumerate(attendees)] + self._save_attendees(meeting,committee_attendees) + attendees_message = "inserted attendees" + + return attendees_ok, attendees_message + def _handle_datapackage_item(self, meeting_data): text_updated, text_message = None, None parts_updated, parts_message = None, None attended_updated, attended_message = None, None + #sorry for this redundancy, i don't want to break stuff (@beber) + attendees_updated, attendees_message = None, None text_file_path = self._get_datapackage_resource_path(meeting_data["text"]) parts_file_path = self._get_datapackage_resource_path(meeting_data["parts"]) + attendees_file_path = self._get_datapackage_resource_path(meeting_data["attendees"]) cnt, qs = self._get_meetings(meeting_data["committee_id"], meeting_data["meeting_id"]) - ok, error = self._validate_datapackage_item(text_file_path, parts_file_path, cnt) + ok, error = self._validate_datapackage_item(text_file_path, parts_file_path, attendees_file_path, cnt) if ok: meeting = qs.first() text_updated, text_message = self._update_protocol_text(meeting, text_file_path) if not text_updated: parts_updated, parts_message, attended_updated, attended_message = False, "protocol text not updated, so skipping parts updating as well", None, None else: - parts_updated, parts_message, attended_updated, attended_message = self._update_protocol_parts(meeting, - parts_file_path) - return ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attended_updated, attended_message + parts_updated, parts_message, attended_updated, attended_message = self._update_protocol_parts(meeting, parts_file_path) + + attendees_updated, attendees_message = self._update_attendees(meeting, attendees_file_path) + return ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attendees_updated, attendees_message + + def log_return_value(self, ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attendees_updated, attendees_message): - def log_return_value(self, ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, - attended_updated, attended_message): if ok: self.logger.debug(text_message) self.logger.debug(parts_message) @@ -128,9 +154,11 @@ def log_return_value(self, ok, error, meeting_data, text_updated, text_message, self.logger.error("committee {} meeting {}: updated committee meeting protocol parts only".format( meeting_data["committee_id"], meeting_data["meeting_id"])) else: - self.logger.debug( - "committee {} meeting {}: no update to protocol text or parts".format(meeting_data["committee_id"], - meeting_data["meeting_id"])) - else: - self.logger.info("committee {} meeting {}: error scraping committee meeting protocol: {}".format( - meeting_data["committee_id"], meeting_data["meeting_id"], error)) + self.logger.debug("committee {} meeting {}: no update to protocol text or parts".format(meeting_data["committee_id"], meeting_data["meeting_id"])) + + #fuck this spaghetti (@beber) + if attendees_updated: + self.logger.info("committee {} meeting {}: updated meeting attendees".format(meeting_data["committee_id"], meeting_data["meeting_id"])) + else: + self.logger.debug("committee {} meeting {}: no update to meeting attendees".format(meeting_data["committee_id"], meeting_data["meeting_id"])) + diff --git a/knesset_data_django/common/scrapers/root_datapackage_scraper.py b/knesset_data_django/common/scrapers/root_datapackage_scraper.py index 6aa467a..70f5738 100644 --- a/knesset_data_django/common/scrapers/root_datapackage_scraper.py +++ b/knesset_data_django/common/scrapers/root_datapackage_scraper.py @@ -178,9 +178,14 @@ def log_scrape_return_value(self, scrape_classes_return_value): scraper_instance, scrape_return_values = scrape_class_return_value if scraper_instance: i = 0 - for scrape_return_value in scrape_return_values: - scraper_instance.log_return_value(*scrape_return_value) - i += 1 + try: + for scrape_return_value in scrape_return_values: + scraper_instance.log_return_value(*scrape_return_value) + i += 1 + except Exception as e: + message = "unexpected exception parsing return value for {}".format(scraper_class) + self.logger.exception(message) + raise Exception(message) self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__)) else: self.logger.debug("skipping scraper {}".format(scraper_class.__name__)) diff --git a/knesset_data_django/kns_persons/__init__.py b/knesset_data_django/kns_persons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/models.py b/knesset_data_django/kns_persons/models.py new file mode 100644 index 0000000..b996a5c --- /dev/null +++ b/knesset_data_django/kns_persons/models.py @@ -0,0 +1,45 @@ +from django.db import models + + +class KnessetPerson(models.Model): + source_id = models.IntegerField() + last_name = models.CharField(max_length=64) + first_name = models.CharField(max_length=64) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + email = models.EmailField(blank=True, null=True) + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetPosition(models.Model): + source_id = models.IntegerField() + description = models.CharField(max_length=256) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + last_update = models.DateTimeField() + + +class KnessetPersonToPosition(models.Model): + source_id = models.IntegerField() + person_id = models.IntegerField() + position_id = models.IntegerField() + knesset_num = models.IntegerField() + ministry_id = models.IntegerField() + ministry_name = models.CharField(max_length=256) + duty_description = models.CharField(max_length=256) + faction_id = models.IntegerField() + faction_name = models.CharField(max_length=64) + gov_num = models.IntegerField() + committee_id = models.IntegerField() + committee_name = models.CharField(max_length=64) + start_update = models.DateTimeField() + finish_update = models.DateTimeField() + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetSiteCode(models.Model): + source_id = models.IntegerField() + kns_id = models.IntegerField() + site_id = models.IntegerField() diff --git a/knesset_data_django/kns_persons/scrapers/__init__.py b/knesset_data_django/kns_persons/scrapers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/scrapers/person.py b/knesset_data_django/kns_persons/scrapers/person.py new file mode 100644 index 0000000..9f210a1 --- /dev/null +++ b/knesset_data_django/kns_persons/scrapers/person.py @@ -0,0 +1,56 @@ +from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper +from knesset_data_django.common.exceptions import TooManyObjectsException +from ..models import KnessetPerson + + +class PersonScraper(BaseDatapackageScraper): + DATAPACKAGE_RESOURCE_NAME = "person" + + def _handle_datapackage_item(self, person_data): + """ + updates or create a committee object based on dataservice_person + :param person_data: dataservice person object + :return: tuple(person, created) the updated or created person model object and True/False if it was created + """ + person_knesset_id = person_data["id"] + person_model_data = { + "last_name": person_data["last_name"], + "first_name": person_data["first_name"], + "gender_id": person_data["gender_id"], + "gender_description": person_data["gender_description"], + "email": person_data["email"], + "is_current": person_data["is_current"], + "last_update": person_data["last_update"] + } + person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id) + person_qs_count = person_qs.count() + if person_qs_count == 1: + person = person_qs.first() + needs_update = False + for attr, scraped_value in person_model_data.iteritems(): + db_value = getattr(person, attr) + if db_value != scraped_value: + needs_update = True + break + if needs_update: + [setattr(person, k, v) for k, v in person_model_data.iteritems()] + created, updated, message = False, True, "detected a change in one of the fields, updating person" + else: + created, updated, message = False, False, "existing person in DB, no change" + elif person_qs_count == 0: + person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data) + created, updated, message = True, False, "created person" + else: + raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id, + [c.id for c in + person_qs])) + if updated or created: + person.save() + return person, created, updated, message + + def log_return_value(self, person, created, updated, message): + prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name) + if created or updated: + self.logger.info(u"{}: {}".format(prefix, message)) + else: + self.logger.debug(u'{}: {}'.format(prefix, message)) diff --git a/knesset_data_django/persons/models.py b/knesset_data_django/persons/models.py index df339e2..bc64b57 100644 --- a/knesset_data_django/persons/models.py +++ b/knesset_data_django/persons/models.py @@ -1,2 +1,2 @@ # currently uses Open Knesset models. TODO: move to a separate data access layer -from persons.models import Person, PersonAlias \ No newline at end of file +from persons.models import Person, PersonAlias